From 780526041a61ae9f9738be86b9aa92505ee866bd Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 8 Dec 2021 16:19:03 +0100 Subject: [PATCH 01/72] bumping version number --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0e7e7a847..a38a0ead3 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.6.0' + version = '3.7.0-SNAPSHOT' apply plugin: 'java-library' apply plugin: 'maven-publish' From d10d91430e2b4d16e0f8ca5d3e1e98d7f44edead Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 8 Dec 2021 16:51:29 +0100 Subject: [PATCH 02/72] Added ThreadLocalObjectCache to reuse objects in a thread-safe manner --- .../cineast/core/features/OCRSearch.java | 23 +++++++++++++--- .../core/util/ThreadLocalObjectCache.java | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/ThreadLocalObjectCache.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java index 5eb85bf6b..e104c76ea 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java @@ -2,6 +2,7 @@ import boofcv.io.image.ConvertBufferedImage; import boofcv.struct.image.GrayU8; +import com.google.common.cache.CacheLoader; import georegression.struct.shapes.Quadrilateral_F64; import org.apache.commons.text.similarity.JaroWinklerSimilarity; import org.opencv.core.CvType; @@ -15,8 +16,8 @@ import java.awt.*; import java.awt.image.BufferedImage; -import java.util.*; import java.util.List; +import java.util.*; /** * OCR is handled by adding fuzziness / levenshtein-distance support to the query if there are no quotes present (as quotes indicate precision) This makes sense here since we expect small errors from OCR sources @@ -50,6 +51,20 @@ public class OCRSearch extends AbstractTextRetriever { private static final MultiTracker.TRACKER_TYPE tracker_type = MultiTracker.TRACKER_TYPE.CIRCULANT; private static final int threshold_stream_length = 9; + private static final ThreadLocalObjectCache detectorCache = new ThreadLocalObjectCache<>(new CacheLoader() { + @Override + public TextDetector_EAST load(Thread key) { + return new TextDetector_EAST().initialize(); + } + }); + + private static final ThreadLocalObjectCache recognizerCache = new ThreadLocalObjectCache<>(new CacheLoader() { + @Override + public TextRecognizer_CTC load(Thread key) { + return new TextRecognizer_CTC().initialize(); + } + }); + public OCRSearch() { super(OCR_TABLE_NAME); } @@ -149,7 +164,7 @@ private Mat img2Mat(BufferedImage original) { data = new byte[in.getWidth() * in.getHeight() * (int) out.elemSize()]; int[] dataBuff = in.getRGB(0, 0, in.getWidth(), in.getHeight(), null, 0, in.getWidth()); for (int i = 0; i < dataBuff.length; i++) { - data[i * 3] = (byte) ((dataBuff[i] >> 0) & 0xFF); + data[i * 3] = (byte) ((dataBuff[i]) & 0xFF); data[i * 3 + 1] = (byte) ((dataBuff[i] >> 8) & 0xFF); data[i * 3 + 2] = (byte) ((dataBuff[i] >> 16) & 0xFF); } @@ -262,8 +277,8 @@ public double getAverageIntersectionOverUnion(List coordinate */ @Override public void processSegment(SegmentContainer shot) { - TextDetector_EAST detector = new TextDetector_EAST().initialize(); - TextRecognizer_CTC recognizer = new TextRecognizer_CTC().initialize(); + TextDetector_EAST detector = detectorCache.get(); + TextRecognizer_CTC recognizer = recognizerCache.get(); int lenVideo = shot.getVideoFrames().size(); // Scene text extraction for image diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ThreadLocalObjectCache.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ThreadLocalObjectCache.java new file mode 100644 index 000000000..3ce87285a --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ThreadLocalObjectCache.java @@ -0,0 +1,26 @@ +package org.vitrivr.cineast.core.util; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +public class ThreadLocalObjectCache { + + private final LoadingCache cache; + + public ThreadLocalObjectCache(CacheLoader cacheLoader) { + this.cache = CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES).build(cacheLoader); + } + + public T get() { + try { + return this.cache.get(Thread.currentThread()); + } catch (ExecutionException executionException) { + throw new IllegalStateException("Error accessing ThreadLocalObjectCache for thread " + Thread.currentThread().getName()); + } + } + +} From 201edae3cb900ac0b702d3c563991b342f679152 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Wed, 8 Dec 2021 10:19:52 +0100 Subject: [PATCH 03/72] Implemented visual-text co-embedding extraction of full video segments rather than only the most representative frame. --- .../core/features/VisualTextCoEmbedding.java | 93 ++++++++++++++++++- .../abstracts/MotionHistogramCalculator.java | 12 +-- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java index 0bb835012..156873777 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.ndarray.NdArrays; @@ -29,6 +30,7 @@ public class VisualTextCoEmbedding extends AbstractFeatureModule { private static final int EMBEDDING_SIZE = 256; + private static final int ENCODING_SIZE = 1536; private static final String TABLE_NAME = "features_visualtextcoembedding"; private static final Distance DISTANCE = ReadableQueryConfig.Distance.euclidean; @@ -87,15 +89,27 @@ public VisualTextCoEmbedding() { @Override public void processSegment(SegmentContainer shot) { - if (shot.getMostRepresentativeFrame() == VideoFrame.EMPTY_VIDEO_FRAME) { + // Return if already processed + if (phandler.idExists(shot.getId())) { return; } - BufferedImage image = shot.getMostRepresentativeFrame().getImage().getBufferedImage(); + if (!(shot.getVideoFrames().size() > 0 && shot.getVideoFrames().get(0) == VideoFrame.EMPTY_VIDEO_FRAME)) { + // Segment is video + List frames = shot.getVideoFrames().stream() + .map(frame -> frame.getImage().getBufferedImage()) + .collect(Collectors.toList()); - if (image != null) { - float[] embeddingArray = embedImage(image); + float[] embeddingArray = embedVideo(frames); this.persist(shot.getId(), new FloatVectorImpl(embeddingArray)); + } else if (shot.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { + // Segment is image + BufferedImage image = shot.getMostRepresentativeFrame().getImage().getBufferedImage(); + + if (image != null) { + float[] embeddingArray = embedImage(image); + this.persist(shot.getId(), new FloatVectorImpl(embeddingArray)); + } } } @@ -203,6 +217,77 @@ private float[] embedImage(BufferedImage image) { } } + private float[] embedVideo(List frames) { + initializeVisualEmbedding(); + + List encodings = frames.stream().map(this::encodeImage).collect(Collectors.toList()); + + // Sum + float[] meanEncoding = encodings.stream().reduce(new float[ENCODING_SIZE], (encoding0, encoding1) -> { + float[] tempSum = new float[ENCODING_SIZE]; + + for (int i = 0; i < ENCODING_SIZE; i++) { + tempSum[i] = encoding0[i] + encoding1[i]; + } + + return tempSum; + }); + + // Calculate mean + for (int i = 0; i < ENCODING_SIZE; i++) { + meanEncoding[i] /= encodings.size(); + } + + try (TFloat32 encoding = TFloat32.tensorOf(Shape.of(1, ENCODING_SIZE), DataBuffers.of(meanEncoding))) { + HashMap inputMap = new HashMap<>(); + + inputMap.put(VISUAL_CO_EMBEDDING_INPUT, encoding); + + Map resultMap = visualCoEmbedding.call(inputMap); + try (TFloat32 embedding = (TFloat32) resultMap.get(VISUAL_CO_EMBEDDING_OUTPUT)) { + + float[] embeddingArray = new float[EMBEDDING_SIZE]; + FloatDataBuffer floatBuffer = DataBuffers.of(embeddingArray); + // Beware TensorFlow allows tensor writing to buffers through the function read rather than write + embedding.read(floatBuffer); + + return embeddingArray; + } + } + } + + /** + * Encodes the given image using the encoding network. + *

+ * Visual embedding must already be initialized. + * + * @return Intermediary encoding, not yet embedded. + */ + private float[] encodeImage(BufferedImage image) { + if (image.getWidth() != IMAGE_WIDTH || image.getHeight() != IMAGE_HEIGHT) { + image = rescale(image, IMAGE_WIDTH, IMAGE_HEIGHT); + } + int[] colors = image.getRGB(0, 0, IMAGE_WIDTH, IMAGE_HEIGHT, null, 0, IMAGE_WIDTH); + int[] rgb = colorsToRGB(colors); + float[] processedColors = preprocessInput(rgb); + + try (TFloat32 imageTensor = TFloat32.tensorOf(Shape.of(1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), DataBuffers.of(processedColors))) { + HashMap inputMap = new HashMap<>(); + inputMap.put(VISUAL_EMBEDDING_INPUT, imageTensor); + + Map resultMap = visualEmbedding.call(inputMap); + + try (TFloat32 encoding = (TFloat32) resultMap.get(VISUAL_EMBEDDING_OUTPUT)) { + + float[] embeddingArray = new float[ENCODING_SIZE]; + FloatDataBuffer floatBuffer = DataBuffers.of(embeddingArray); + encoding.read(floatBuffer); + + return embeddingArray; + } + } + } + /** * Preprocesses input in a way equivalent to that performed in the Python TensorFlow library. *

diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java index 19ebf5002..be4acf10e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java @@ -88,8 +88,8 @@ protected Pair, ArrayList>> getSubDivHist( for (int i = 0; i < sums.length; ++i) { float[] hist = hists[i]; double sum = 0; - for (int j = 0; j < hist.length; ++j) { - sum += hist[j]; + for (float v : hist) { + sum += v; } if (sum > 0) { for (int j = 0; j < hist.length; ++j) { @@ -100,22 +100,22 @@ protected Pair, ArrayList>> getSubDivHist( sums[i] = sum; } - ArrayList sumList = new ArrayList(sums.length); + ArrayList sumList = new ArrayList<>(sums.length); for (double d : sums) { sumList.add(d); } - ArrayList> histList = new ArrayList>( + ArrayList> histList = new ArrayList<>( hists.length); for (float[] hist : hists) { - ArrayList h = new ArrayList(8); + ArrayList h = new ArrayList<>(8); for (float f : hist) { h.add(f); } histList.add(h); } - return new Pair, ArrayList>>(sumList, + return new Pair<>(sumList, histList); } From 1ecf39ae0f0bc6272d55ff6336dec9bd7d801d97 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 9 Dec 2021 10:24:09 +0100 Subject: [PATCH 04/72] Implemented early return to conform to codebase guidelines. --- .../cineast/core/features/VisualTextCoEmbedding.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java index 156873777..496a7517a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java @@ -94,22 +94,28 @@ public void processSegment(SegmentContainer shot) { return; } + // Case: segment contains video frames if (!(shot.getVideoFrames().size() > 0 && shot.getVideoFrames().get(0) == VideoFrame.EMPTY_VIDEO_FRAME)) { - // Segment is video List frames = shot.getVideoFrames().stream() .map(frame -> frame.getImage().getBufferedImage()) .collect(Collectors.toList()); float[] embeddingArray = embedVideo(frames); this.persist(shot.getId(), new FloatVectorImpl(embeddingArray)); - } else if (shot.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { - // Segment is image + + return; + } + + // Case: segment contains image + if (shot.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { BufferedImage image = shot.getMostRepresentativeFrame().getImage().getBufferedImage(); if (image != null) { float[] embeddingArray = embedImage(image); this.persist(shot.getId(), new FloatVectorImpl(embeddingArray)); } + + // Insert return here if additional cases are added! } } From f774eb9ce3c5865e2b9a9f2d7677a7fb8b3694c4 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 9 Dec 2021 15:29:27 +0100 Subject: [PATCH 05/72] Refactored visual-text co-embedding to load buffered image only once needed. --- .../cineast/core/features/VisualTextCoEmbedding.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java index 496a7517a..0486366c0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java @@ -20,6 +20,7 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; @@ -96,8 +97,8 @@ public void processSegment(SegmentContainer shot) { // Case: segment contains video frames if (!(shot.getVideoFrames().size() > 0 && shot.getVideoFrames().get(0) == VideoFrame.EMPTY_VIDEO_FRAME)) { - List frames = shot.getVideoFrames().stream() - .map(frame -> frame.getImage().getBufferedImage()) + List frames = shot.getVideoFrames().stream() + .map(VideoFrame::getImage) .collect(Collectors.toList()); float[] embeddingArray = embedVideo(frames); @@ -223,10 +224,10 @@ private float[] embedImage(BufferedImage image) { } } - private float[] embedVideo(List frames) { + private float[] embedVideo(List frames) { initializeVisualEmbedding(); - List encodings = frames.stream().map(this::encodeImage).collect(Collectors.toList()); + List encodings = frames.stream().map(image -> encodeImage(image.getBufferedImage())).collect(Collectors.toList()); // Sum float[] meanEncoding = encodings.stream().reduce(new float[ENCODING_SIZE], (encoding0, encoding1) -> { From e442ec553905774a8c8c57f09d8100fd3b08639d Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 16 Dec 2021 20:52:45 +0100 Subject: [PATCH 06/72] Removed unused and empty JavaDoc. --- .../org/vitrivr/cineast/api/APIEndpoint.java | 3 -- .../cineast/api/rest/OpenApiCompatHelper.java | 6 --- .../handlers/interfaces/GetRestHandler.java | 2 - .../queries/AbstractQueryMessageHandler.java | 1 - .../vitrivr/cineast/core/config/IdConfig.java | 1 - .../vitrivr/cineast/core/data/GpsData.java | 3 -- .../cineast/core/data/UniqueElementGrid.java | 6 +-- .../entities/SimpleFeatureDescriptor.java | 5 -- .../core/data/frames/AudioDescriptor.java | 4 -- .../core/data/frames/VideoDescriptor.java | 25 ---------- .../vitrivr/cineast/core/data/m3d/Mesh.java | 22 +-------- .../cineast/core/data/m3d/VoxelGrid.java | 2 - .../core/data/providers/DurationProvider.java | 12 ----- .../containers/AudioQueryTermContainer.java | 10 +--- .../core/data/raw/images/MultiImage.java | 7 --- .../core/data/segments/AudioSegment.java | 36 +------------- .../core/data/segments/ImageSegment.java | 15 ------ .../core/data/segments/Model3DSegment.java | 15 ------ .../core/data/segments/VideoSegment.java | 26 ---------- .../vitrivr/cineast/core/db/DBSelector.java | 1 - .../db/adampro/ADAMproMessageBuilder.java | 49 ------------------- .../dao/reader/DatabaseLookupException.java | 14 ------ .../core/db/dao/writer/MediaObjectWriter.java | 7 --- .../core/db/setup/AttributeDefinition.java | 16 ------ .../extraction/decode/general/Decoder.java | 2 - .../decode/image/DefaultImageDecoder.java | 4 -- .../decode/m3d/ModularMeshDecoder.java | 2 - .../extraction/decode/m3d/OBJMeshDecoder.java | 2 - .../extraction/decode/m3d/OFFMeshDecoder.java | 2 - .../extraction/decode/m3d/STLMeshDecoder.java | 2 - .../decode/subtitle/cc/CCSubTitleDecoder.java | 9 ---- .../decode/video/FFMpegVideoDecoder.java | 7 --- .../SequentialObjectIdGenerator.java | 1 - .../idgenerator/UniqueObjectIdGenerator.java | 1 - .../audio/ConstantLengthAudioSegmenter.java | 2 - .../segmenter/general/Segmenter.java | 2 - .../image/ImageSequenceSegmenter.java | 4 -- .../video/VideoHistogramSegmenter.java | 18 ------- .../core/features/AudioFingerprint.java | 10 ---- .../core/features/HOGMirflickr25K256.java | 2 - .../core/features/HOGMirflickr25K512.java | 2 - .../cineast/core/features/Lightfield.java | 19 ------- .../cineast/core/features/MFCCShingle.java | 4 -- .../cineast/core/features/MelodyEstimate.java | 6 --- .../vitrivr/cineast/core/features/SURF.java | 3 -- .../core/features/SURFMirflickr25K256.java | 2 - .../core/features/SURFMirflickr25K512.java | 2 - .../core/features/ShapeCentroidDistance.java | 10 ---- .../core/features/SphericalHarmonics.java | 7 --- .../AbstractCodebookFeatureModule.java | 13 +---- .../codebook/ImageCodebookGenerator.java | 5 -- .../vitrivr/cineast/core/util/ColorUtils.java | 2 - .../core/util/ImageHistogramEqualizer.java | 2 - .../cineast/core/util/MotionHistoryImage.java | 3 -- .../cineast/core/util/OptionalUtil.java | 12 ++--- .../cineast/core/util/ReflectionHelper.java | 2 - .../vitrivr/cineast/core/util/TextStream.java | 4 -- .../vitrivr/cineast/core/util/audio/MFCC.java | 7 --- .../cineast/core/util/audio/pitch/Pitch.java | 2 - .../pitch/estimation/KLF0PitchEstimator.java | 7 +-- .../audio/pitch/tracking/PitchContour.java | 2 - .../audio/pitch/tracking/PitchTracker.java | 26 ---------- .../core/util/dsp/SamplingUtilities.java | 1 - .../cineast/core/util/dsp/fft/FFTUtil.java | 2 - .../cineast/core/util/dsp/fft/STFT.java | 26 ---------- .../cineast/core/util/dsp/fft/Spectrum.java | 10 ---- .../visualization/AudioSignalVisualizer.java | 1 - .../cineast/core/util/images/SURFHelper.java | 2 - .../core/util/images/ZernikeHelper.java | 3 -- .../cineast/core/util/json/JsonWriter.java | 3 -- .../functions/SphericalHarmonicsFunction.java | 3 -- .../math/functions/ZernikeBasisFunction.java | 6 --- .../core/util/mesh/MeshColoringUtil.java | 4 -- .../cineast/core/util/web/ImageParser.java | 6 --- .../core/util/dsp/fft/WindowFunctionTest.java | 5 -- .../evaluation/EvaluationResult.java | 10 ---- .../importer/PlainTextImporter.java | 1 - .../importer/TagsFulltextImporter.java | 1 - .../standalone/importer/VbsMetaImporter.java | 1 - .../handlers/AsrDataImportHandler.java | 10 ---- .../handlers/JsonDataImportHandler.java | 5 -- .../handlers/OcrDataImportHandler.java | 9 ---- .../handlers/ProtoDataImportHandler.java | 5 -- .../importer/lsc2020/LSCUtilities.java | 7 --- .../importer/lsc2020/MyscealTagImporter.java | 1 - .../lsc2020/ProcessingMetaImporter.java | 5 +- .../run/ExtractionItemContainer.java | 2 - .../runtime/ExtractionPipeline.java | 2 - 88 files changed, 13 insertions(+), 618 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index 5f00442d1..e13ce1a7c 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -439,9 +439,6 @@ private void registerRestOperations() { /** * If configured, this registers two special routes that serve the media objects as media content and additionally a thumbnails endpoint for them. - * - * @param service - * @param config */ private void registerServingRoutes(final Javalin service, final APIConfig config) { if (config.getServeContent()) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java index 917af57e0..dceddd721 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java @@ -52,9 +52,6 @@ private OpenApiCompatHelper() { /** * Creates the Javalin options used to create an OpenAPI specification. - * - * @param config - * @return */ public static OpenApiOptions getJavalinOpenApiOptions(APIConfig config) { //Default Javalin JSON mapper includes all null values, which breakes the openapi specs. @@ -76,9 +73,6 @@ public static OpenApiOptions getJavalinOpenApiOptions(APIConfig config) { /** * Creates the base {@link OpenAPI} specification. - * - * @param config - * @return */ public static OpenAPI getOpenApi(APIConfig config) { OpenAPI api = new OpenAPI(); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/interfaces/GetRestHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/interfaces/GetRestHandler.java index be01a325b..037842f32 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/interfaces/GetRestHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/interfaces/GetRestHandler.java @@ -25,8 +25,6 @@ default void handle(@NotNull Context ctx) throws Exception { /** * Performs the GET REST operation of this handler and sends the result to the requester. Exception handling has to be done by the caller. - * - * @param ctx */ default void get(Context ctx) { ctx.json(doGet(ctx)); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java index cee9a4259..147899f23 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java @@ -324,7 +324,6 @@ protected List submitMetadata(Session session, String queryId, List score). - * @return */ protected List> finalizeAndSubmitResults(Session session, String queryId, String category, int containerId, List raw) { StopWatch watch = StopWatch.createStarted(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java index fdd090313..230a9c497 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java @@ -54,7 +54,6 @@ public IdConfig() { * * @param name Name of the {@link ObjectIdGenerator} * @param existenceCheckMode Determines the 'existence check mode' for objectId's of {@link MediaObjectDescriptor}s - * @param properties */ @JsonCreator public IdConfig(@JsonProperty(value = "name", required = true) String name, diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java index 29500ac85..e438259d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java @@ -147,9 +147,6 @@ public static GpsData ofJson(Path file) { *

  • the minutes or seconds, 0 is used for this
    • * * Following this approach, the date is somewhat 'centered' within a year, so that the {@link org.vitrivr.cineast.core.features.TemporalDistance} works best. - * - * @param dating - * @return */ public static Optional parseFuzzyDating(String dating){ if(dating == null){ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java index 8a8d9e86f..1256cdf3c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java @@ -33,11 +33,7 @@ public T get(Position p){ } /** - * returns the grid position of the specified element or null if the element is not part of the - * map - * - * @param element - * @return + * Returns the grid position of the specified element or null if the element is not part of the map. */ public Position getPosition(T element) { return map.inverse().get(element); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java index fdad55462..3c0677b07 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java @@ -10,11 +10,6 @@ public class SimpleFeatureDescriptor { /** */ public final ReadableFloatVector feature; - /** - * - * @param segmentId - * @param feature - */ public SimpleFeatureDescriptor(String segmentId, ReadableFloatVector feature) { this.segmentId = segmentId; this.feature = feature; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java index a1aecaab2..16859943c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java @@ -17,10 +17,6 @@ public class AudioDescriptor { /** * Constructor for an AudioDescriptor. - * - * @param samplingrate - * @param channels - * @param duration */ public AudioDescriptor(float samplingrate, int channels, long duration) { this.samplingrate = samplingrate; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java index 7765a013d..462b3f36e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java @@ -20,11 +20,6 @@ public class VideoDescriptor { /** * Constructor for VideoDescriptor - * - * @param fps - * @param duration - * @param width - * @param height */ public VideoDescriptor(float fps, long duration, int width, int height) { this.fps = fps; @@ -33,38 +28,18 @@ public VideoDescriptor(float fps, long duration, int width, int height) { this.height = height; } - /** - * Getter for fps. - * - * @return - */ public float getFps() { return fps; } - /** - * Getter for duration. - * - * @return - */ public long getDuration() { return duration; } - /** - * Getter for width. - * - * @return - */ public int getWidth() { return width; } - /** - * Getter for height - * - * @return - */ public int getHeight() { return height; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java index 406cbf486..baaeeab65 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java @@ -32,20 +32,10 @@ public class Vertex { /** List of faces the current vertex participates in. */ private final List faces = new ArrayList<>(4); - /** - * - * @param position - */ public Vertex(Vector3f position) { this(position, new Vector3f(0.0f, 0.0f, 0.0f), new Vector3f(0.0f, 0.0f, 0.0f)); } - /** - * - * @param position - * @param normal - * @param color - */ public Vertex(Vector3f position,Vector3f color, Vector3f normal) { this.position = position; this.normal = normal; @@ -159,8 +149,6 @@ public final FaceType getType() { /** * Constructor for a face. - * - * @param indices */ private Face(Vector4i indices) { /* If the w-index is greater than -1 a QUAD face is created. */ @@ -269,8 +257,6 @@ public double area() { /** * Calculates and returns the face normal. - * - * @return */ public Vector3f normal() { Vector3f e1 = new Vector3f(this.vertices[1].position).sub(this.vertices[0].position); @@ -353,17 +339,13 @@ public Mesh(int faces, int vertices) { /** * Adds an vector defining a vertex to the Mesh. - * - * @param vertex */ public synchronized void addVertex(Vector3f vertex) { this.addVertex(vertex, new Vector3f(1.0f, 1.0f, 1.0f)); } /** - * Adds an vector defining a vertex to the Mesh. - * - * @param vertex + * Adds a vector defining a vertex to the Mesh. */ public synchronized void addVertex(Vector3f vertex, Vector3f color) { this.addVertex(vertex, color, new Vector3f(0.0f, 0.0f, 0.0f)); @@ -371,8 +353,6 @@ public synchronized void addVertex(Vector3f vertex, Vector3f color) { /** * Adds an vector defining a vertex to the Mesh. - * - * @param vertex */ public synchronized void addVertex(Vector3f vertex, Vector3f color, Vector3f normal) { this.vertices.add(new Vertex(vertex, color, normal)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java index ccf5e3bb0..9fd524aa1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java @@ -160,8 +160,6 @@ public final int getInvisible() { /** * Returns true if VoxelGrid is visible (i.e. there is at least one * visible Voxel) and false otherwise. - * - * @return */ public final boolean isVisible() { return this.visible > 0; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java index e8e711207..b066cf5c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java @@ -4,8 +4,6 @@ public interface DurationProvider { /** * Returns the start in some arbitrary unit, e.g. in samples or frames depending * on the implementation. - * - * @return */ default int getStart(){ return 0; @@ -14,8 +12,6 @@ default int getStart(){ /** * Returns the end in some arbitrary unit, e.g. in samples or frames depending * on the implementation. - * - * @return */ default int getEnd(){ return 0; @@ -23,8 +19,6 @@ default int getEnd(){ /** * Returns the relative start in percent (value between 0.0 and 1.0) - * - * @return */ default float getRelativeStart(){ return 0f; @@ -32,8 +26,6 @@ default float getRelativeStart(){ /** * Returns the relative end in percent (value between 0.0 and 1.0) - * - * @return */ default float getRelativeEnd(){ return 0f; @@ -41,8 +33,6 @@ default float getRelativeEnd(){ /** * Returns the absolute start in seconds. - * - * @return */ default float getAbsoluteStart(){ return 0f; @@ -50,8 +40,6 @@ default float getAbsoluteStart(){ /** * Returns the absolute end in seconds. - * - * @return */ default float getAbsoluteEnd(){ return 0f; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java index 6af04ef60..e4f58d854 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java @@ -37,7 +37,7 @@ public AudioQueryTermContainer(String data) { * Returns a list of audio-frames contained in the AudioSegment. The default implementation returns * a list containing one, empty frame. * - * @return List auf audio-frames in the audio-segment. + * List auf audio-frames in the audio-segment. */ public AudioQueryTermContainer(List frames) { this.frames = frames; @@ -53,10 +53,6 @@ public AudioQueryTermContainer(List frames) { } } - /** - * - * @return - */ @Override public List getAudioFrames() { return this.frames; @@ -64,8 +60,6 @@ public List getAudioFrames() { /** * Getter for the total number of samples in the AudioSegment. - * - * @return */ @Override public int getNumberOfSamples() { @@ -74,8 +68,6 @@ public int getNumberOfSamples() { /** * Getter for the total duration of the AudioSegment. - * - * @return */ @Override public float getAudioDuration() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java index aed60ebdc..7026daed3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java @@ -70,13 +70,6 @@ public void clear() {} void clear(); - /** - * - * @param width - * @param height - * @param colors - * @return - */ static int checkHeight(int width, int height, int[] colors){ if(colors.length / width != height){ height = colors.length / width; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java index 63e9a32d8..cd736c83e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java @@ -15,10 +15,7 @@ * * The AudioSegment implements the SegmentContainer interface and provides access to different, frames-related data. * - * - * @TODO: - * - Perform basic checks when adding an audio-frame (sample-rate, duration...) - * + * TODO: Perform basic checks when adding an audio-frame (sample-rate, duration...) */ public class AudioSegment implements SegmentContainer { /** Segment ID of the AudioSegment. */ @@ -50,27 +47,16 @@ public String getId() { return this.segmentId; } - /** - * @param id - * @return a unique id of this - */ @Override public void setId(String id) { this.segmentId = id; } - /** - * - * @return - */ @Override public String getSuperId() { return this.objectId; } - /** - * @param id - */ @Override public void setSuperId(String id) { this.objectId = id; @@ -113,8 +99,6 @@ public boolean addFrame(AudioFrame frame) { /** * Getter for the total number of samples in the AudioSegment. - * - * @return */ @Override public int getNumberOfSamples() { @@ -123,27 +107,17 @@ public int getNumberOfSamples() { /** * Getter for the total duration of the AudioSegment. - * - * @return */ @Override public float getAudioDuration() { return totalDuration; } - /** - * - * @return - */ @Override public float getSamplingrate() { return this.descriptor.getSamplingrate(); } - /** - * - * @return - */ @Override public int getChannels() { return this.descriptor.getChannels(); @@ -151,8 +125,6 @@ public int getChannels() { /** * Getter for the frame-number of the start-frame. - * - * @return */ @Override public int getStart(){ @@ -165,8 +137,6 @@ public int getStart(){ /** * Getter for the frame-number of the end-frame. - * - * @return */ @Override public int getEnd(){ @@ -179,8 +149,6 @@ public int getEnd(){ /** * Getter for the start (in seconds) of this segment. - * - * @return */ @Override public float getAbsoluteStart(){ @@ -193,8 +161,6 @@ public float getAbsoluteStart(){ /** * Getter for the end (in seconds) of this segment. - * - * @return */ @Override public float getAbsoluteEnd(){ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java index eb1e89bb9..25a2afe67 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java @@ -39,10 +39,6 @@ public String getId() { return this.segmentId; } - /** - * @param id - * @return a unique id of this - */ @Override public void setId(String id) { this.segmentId = id; @@ -53,9 +49,6 @@ public String getSuperId() { return this.objectId; } - /** - * @param id - */ @Override public void setSuperId(String id) { this.objectId = id; @@ -63,8 +56,6 @@ public void setSuperId(String id) { /** * Returns the median image, which is the actual image. - * - * @return */ @Override public MultiImage getAvgImg() { @@ -73,8 +64,6 @@ public MultiImage getAvgImg() { /** * Returns the average image, which is the actual image. - * - * @return */ @Override public MultiImage getMedianImg() { @@ -83,8 +72,6 @@ public MultiImage getMedianImg() { /** * Returns a single frame - the image. - * - * @return */ @Override public List getVideoFrames() { @@ -95,8 +82,6 @@ public List getVideoFrames() { /** * Returns a most representative frame - the image. - * - * @return */ @Override public VideoFrame getMostRepresentativeFrame() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java index d8b93c7d5..0771fcfff 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java @@ -43,27 +43,16 @@ public final String getId() { return this.segmentId; } - /** - * @param id - * @return a unique id of this - */ @Override public final void setId(String id) { this.segmentId = id; } - /** - * - * @return - */ @Override public final String getSuperId() { return this.objectId; } - /** - * @param id - */ @Override public final void setSuperId(String id) { this.objectId = id; @@ -79,10 +68,6 @@ public final ReadableMesh getMesh() { return this.mesh; } - /** - * - * @return - */ @Override public final WritableMesh getNormalizedMesh() { return this.normalizedMesh; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java index 33ecd72a0..5a3f4dfff 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java @@ -57,9 +57,6 @@ public class VideoSegment implements SegmentContainer { */ public VideoSegment() {} - /** - * @param movieId - */ public VideoSegment(String movieId) { this.movieId = movieId; } @@ -149,8 +146,6 @@ public float getAudioDuration() { /** * Getter for samplingrate of the audio in this {@link VideoSegment}. If the {@link VideoSegment} does not * contain any audio, this method returns 0. - * - * @return */ @Override public float getSamplingrate() { @@ -164,8 +159,6 @@ public float getSamplingrate() { /** * Getter for number of audio channels for the {@link VideoSegment}. If the {@link VideoSegment} does not * contain any audio, this method returns 0. - * - * @return */ @Override public int getChannels() { @@ -297,18 +290,11 @@ public String getSuperId() { return this.movieId; } - /** - * @param id - * @return a unique id of this - */ @Override public void setId(String id) { this.shotId = id; } - /** - * @param id - */ @Override public void setSuperId(String id) { this.movieId = id; @@ -322,8 +308,6 @@ public List getSubtitleItems() { /** * Returns the frame-number of the first frame in the segment (relative to the entire stream). - * - * @return */ @Override public int getStart() { @@ -336,8 +320,6 @@ public int getStart() { /** * Returns the frame-number of the last frame in the segment (relative to the entire stream). - * - * @return */ @Override public int getEnd() { @@ -350,8 +332,6 @@ public int getEnd() { /** * Returns the relative start of the VideoSegment in percent (relative to the entire stream). - * - * @return */ @Override public float getRelativeStart() { @@ -360,8 +340,6 @@ public float getRelativeStart() { /** * Returns the relative end of the VideoSegment in percent (relative to the entire stream). - * - * @return */ @Override public float getRelativeEnd() { @@ -370,8 +348,6 @@ public float getRelativeEnd() { /** * Returns the absolute start of the VideoSegment in seconds (relative to the entire stream). - * - * @return */ @Override public float getAbsoluteStart() { @@ -384,8 +360,6 @@ public float getAbsoluteStart() { /** * Returns the absolute end of the VideoSegment in seconds (relative to the entire stream). - * - * @return */ @Override public float getAbsoluteEnd() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index dcab96fdf..e08f3cdd1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -151,7 +151,6 @@ List> getRows(String fieldName, RelationalOpe * @param conditions conditions which will be linked by AND * @param identifier column upon which the retain operation will be performed if the database layer does not support compound boolean retrieval. * @param projection Which columns shall be selected - * @param qc */ default List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig qc) { HashMap> relevant = new HashMap<>(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java index 295153409..0a80d7c39 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java @@ -60,9 +60,6 @@ public class ADAMproMessageBuilder { /** * Constructs and returns a BatchedQueryMessage from the provided query-messages. - * - * @param queries - * @return */ public BatchedQueryMessage buildBatchedQueryMessage(List queries) { synchronized (bqmBuilder) { @@ -74,15 +71,6 @@ public BatchedQueryMessage buildBatchedQueryMessage(List queries) - /** - * - * @param hints - * @param fb - * @param bqMessage - * @param pMessage - * @param nnqMessage - * @return - */ public QueryMessage buildQueryMessage(Collection hints, FromMessage.Builder fb, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage) { @@ -105,14 +93,6 @@ public QueryMessage buildQueryMessage(Collection hint } } - /** - * - * @param hints - * @param bqMessage - * @param pMessage - * @param nnqMessage - * @return - */ public QueryMessage buildQueryMessage(Collection hints, FromMessage fromMessage, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage) { synchronized (this.qmBuilder) { this.qmBuilder.clear(); @@ -133,11 +113,6 @@ public QueryMessage buildQueryMessage(Collection hint } } - /** - * - * @param entity - * @return - */ public FromMessage buildFromMessage(String entity) { synchronized (this.fromBuilder) { this.fromBuilder.clear(); @@ -163,8 +138,6 @@ public FromMessage buildFromSubExpressionMessage(SubExpressionQueryMessage messa /** * Builds a SubExpressionQueryMessage from a QueryMessage. - * - * @param message */ public SubExpressionQueryMessage buildSubExpressionQueryMessage(QueryMessage message) { synchronized (this.seqmBuilder) { @@ -176,8 +149,6 @@ public SubExpressionQueryMessage buildSubExpressionQueryMessage(QueryMessage mes /** * Builds a SubExpressionQueryMessage from a ExpressionQueryMessage. - * - * @param message */ public SubExpressionQueryMessage buildSubExpressionQueryMessage(ExpressionQueryMessage message) { synchronized (this.seqmBuilder) { @@ -219,7 +190,6 @@ public AdamGrpc.ExpressionQueryMessage buildExpressionQueryMessage(SubExpression * * @param expressions List of SubExpressionQueryMessages * @param operation Set operation used for combining partial results - * @return */ public SubExpressionQueryMessage mergeSubexpressions(List expressions, Operation operation, Map options) { /* If list only contains one SubExpressionQueryMessage then return it. */ @@ -242,12 +212,6 @@ public SubExpressionQueryMessage mergeSubexpressions(List tmp = new ArrayList<>( @@ -352,14 +316,6 @@ public WhereMessage buildWhereMessage(String key, Iterable values, Relat } } - /** - * - * @param column - * @param fvm - * @param k - * @param qc - * @return - */ public NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String column, VectorMessage fvm, int k, ReadableQueryConfig qc) { synchronized (nnqmBuilder) { this.nnqmBuilder.clear(); @@ -374,11 +330,6 @@ public NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String col } } - /** - * - * @param qc - * @return - */ public DistanceMessage buildDistanceMessage(ReadableQueryConfig qc) { if (qc == null) { return manhattan; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/DatabaseLookupException.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/DatabaseLookupException.java index 7de62b58b..a12d1e98e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/DatabaseLookupException.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/DatabaseLookupException.java @@ -1,27 +1,13 @@ package org.vitrivr.cineast.core.db.dao.reader; -/** - * - */ public class DatabaseLookupException extends Exception { private static final long serialVersionUID = 1759949037860773209L; - /** - * Constructor - * - * @param message - */ public DatabaseLookupException(String message) { super(message); } - /** - * Constructor - * - * @param message - * @param cause - */ public DatabaseLookupException(String message, Throwable cause) { super(message, cause); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java index a57afd4df..09e3e21e8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java @@ -7,9 +7,6 @@ public class MediaObjectWriter extends AbstractBatchedEntityWriter { - /** - * @param writer - */ public MediaObjectWriter(PersistencyWriter writer) { super(writer, 1, true); } @@ -23,10 +20,6 @@ protected void init() { this.writer.open(MediaObjectDescriptor.ENTITY); } - /** - * @param entity - * @return - */ @Override protected PersistentTuple generateTuple(MediaObjectDescriptor entity) { return this.writer.generateTuple(entity.getObjectId(), entity.getMediatypeId(), entity.getName(), entity.getPath()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java index 2cab30810..4c9337b12 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java @@ -84,34 +84,18 @@ public AttributeType getType() { return this.type; } - /** - * - * @return - */ public boolean hasHint(String hint) { return this.hints.containsKey(hint); } - /** - * - * @return - */ public Optional getHint(String hint) { return Optional.ofNullable(this.hints.get(hint)); } - /** - * - * @return - */ public String getHintOrDefault(String hint, String defaultValue) { return this.hints.getOrDefault(hint, defaultValue); } - /** - * - * @return - */ public void ifHintPresent(String hint, Consumer consumer) { if (this.hints.containsKey(hint)) { consumer.accept(this.hints.get(hint)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java index c912aca30..cf053d950 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java @@ -50,8 +50,6 @@ public interface Decoder extends AutoCloseable { /** * Returns the total number of content pieces T this decoder can return * for a given file. - * - * @return */ int count(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java index 4debd8f47..89ed1d788 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java @@ -115,8 +115,6 @@ public BufferedImage getNext() { /** * Returns the total number of content pieces T this decoder can return * for a given file. - * - * @return */ @Override public int count() { @@ -125,8 +123,6 @@ public int count() { /** * Returns a list of supported files. - * - * @return */ @Override public Set supportedFiles() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java index 58bc414a9..3f844e460 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java @@ -143,8 +143,6 @@ public AbstractQueryTermContainer convert(Path path) { /** * Returns the total number of content pieces T this decoder can return * for a given file. - * - * @return */ @Override public int count() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java index dccee5c9d..384f9dfc4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java @@ -123,8 +123,6 @@ public Mesh getNext() { /** * Returns the total number of content pieces T this decoder can return for a given file. - * - * @return */ @Override public int count() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java index 5f3ad007f..b529b2f4a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java @@ -159,8 +159,6 @@ public Mesh getNext() { /** * Returns the total number of content pieces T this decoder can return for a given file. - * - * @return */ @Override public int count() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java index 2df0fffd7..a6f03e041 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java @@ -243,8 +243,6 @@ private Mesh readBinary(InputStream is, int skip) throws IOException { /** * Returns the total number of content pieces T this decoder can return for a given file. - * - * @return */ @Override public int count() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java index 3fd8101e6..0ab34415d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java @@ -119,19 +119,10 @@ public int getNumerOfItems() { return this.items.size(); } - /** - * - * @param index - * @return - */ public SubtitleItem get(int index) { return this.items.get(index); } - /** - * - * @return - */ public SubtitleItem getLast() { return this.items.get(pointer); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java index 450a802e0..cc77011f3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java @@ -131,11 +131,6 @@ public class FFMpegVideoDecoder implements Decoder { /** The {@link CachedDataFactory} reference used to create {@link MultiImage} objects. */ private CachedDataFactory factory; - /** - * - * @param queue - * @return - */ private boolean readFrame(boolean queue) { /* Tries to read a new packet from the stream. */ int read_results = avformat.av_read_frame(this.pFormatCtx, this.packet); @@ -644,8 +639,6 @@ public VideoFrame getNext() { /** * Returns the total number of content pieces T this decoder can return for a given file. - * - * @return */ @Override public int count() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java index 78bdc9d40..ed763cb48 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java @@ -51,7 +51,6 @@ public SequentialObjectIdGenerator(Map properties) { * * @param path Path to the file for which an ID should be generated. * @param type MediaType of the file for which an ID should be generated. - * @return */ @Override public String next(Path path, MediaType type) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java index 4ca6ba002..f1f39e2c2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java @@ -44,7 +44,6 @@ public UniqueObjectIdGenerator(Map properties) { * * @param path Path to the file for which an ID should be generated. * @param type MediaType of the file for which an ID should be generated. - * @return */ @Override public String next(Path path, MediaType type) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java index 0c3370b57..fbb69d127 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java @@ -118,8 +118,6 @@ public void init(Decoder decoder, MediaObjectDescriptor object) { * * Important: This method should be designed to block and wait for an appropriate amount of time if the Segmenter * is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. - * - * @return */ @Override public SegmentContainer getNext() throws InterruptedException { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java index 625caa35d..a29dd4796 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java @@ -28,8 +28,6 @@ public interface Segmenter extends Runnable, AutoCloseable { * * Important: This method should be designed to block and wait for an appropriate amount of time if the * Segmenter is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. - * - * @return */ SegmentContainer getNext() throws InterruptedException; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java index c68938175..2797b6d6e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java @@ -53,10 +53,6 @@ public class ImageSequenceSegmenter implements Segmenter { /** Internal flag used to signal that this {@link ImageSequenceSegmenter} is still running. */ private volatile boolean running = false; - /** - * - * @param context - */ public ImageSequenceSegmenter(ExtractionContextProvider context){ this.factory = context.cacheConfig().sharedCachedDataFactory(); if (context.objectIdGenerator() instanceof FileNameObjectIdGenerator) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java index 4d8f4bdcd..c1c1bb793 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java @@ -82,11 +82,6 @@ public VideoHistogramSegmenter(ExtractionContextProvider context, Map decoder, MediaObjectDescriptor } } - /** - * - * @return - */ @Override public SegmentContainer getNext() throws InterruptedException { SegmentContainer nextContainer = this.segments.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); @@ -266,19 +257,10 @@ public void run() { } } - /** - * - * @return - */ private boolean queueFrames(){ return queueFrames(20); } - /** - * - * @param number - * @return - */ private boolean queueFrames(int number) { for(int i = 0; i < number; ++i){ VideoFrame f = this.decoder.getNext(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java index d22d7dc0a..ea1854843 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java @@ -40,10 +40,6 @@ public AudioFingerprint() { super("features_audiofingerprint", 4000.0f, FINGERPRINT); } - /** - * - * @param segment - */ @Override public void processSegment(SegmentContainer segment) { TIntArrayList filteredSpectrum = this.filterSpectrum(segment); @@ -96,7 +92,6 @@ protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig * * @param features A list of feature-vectors (usually generated in the first stage). For each feature, a lookup is executed. May be empty! * @param configs A ReadableQueryConfig object that contains query-related configuration parameters. - * @return */ @Override protected List lookup(List features, List configs) { @@ -171,11 +166,6 @@ protected List generateQueryConfigsForFeatures(ReadableQuer return configs; } - /** - * - * @param segment - * @return - */ private TIntArrayList filterSpectrum(SegmentContainer segment) { /* Prepare empty list of candidates for filtered spectrum. */ TIntArrayList candidates = new TIntArrayList(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java index 00c332a92..d7ca626e6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java @@ -18,8 +18,6 @@ public HOGMirflickr25K256() { /** * Returns the full name of the codebook to use. All codebooks must be placed in the * ./resources/codebooks folder. - * - * @return */ @Override protected String codebook() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java index 306dbd868..79b790e4e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java @@ -18,8 +18,6 @@ public HOGMirflickr25K512() { /** * Returns the full name of the codebook to use. All codebooks must be placed in the * ./resources/codebooks folder. - * - * @return */ @Override protected String codebook() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java index 796fde4f9..c98cb6af3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java @@ -49,12 +49,6 @@ public abstract class Lightfield extends StagedFeatureModule { /** Offscreen rendering environment used to create Lightfield images. */ private final Renderer renderer; - /** - * - * @param tableName - * @param maxDist - * @param camerapositions - */ protected Lightfield(String tableName, float maxDist, int vectorLength, double[][] camerapositions) { super(tableName, maxDist, vectorLength); if (camerapositions.length == 0) { @@ -159,8 +153,6 @@ protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { /** * Processes a single segment. Extracts the mesh and persists all associated features. Segments * that have no mesh or an empty mesh will not be processed. - * - * @param sc */ @Override public void processSegment(SegmentContainer sc) { @@ -232,19 +224,8 @@ protected List featureVectorsFromMesh(ReadableMesh mesh) { return features; } - /** - * - * @param image - * @param poseidx - * @return - */ protected abstract List featureVectorsFromImage(BufferedImage image, int poseidx); - /** - * - * @param poseidx - * @return - */ public double[] positionsForPoseidx(int poseidx) { if (poseidx < this.camerapositions.length) { return this.camerapositions[poseidx]; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java index 8e6949afc..cbe9e7b3e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java @@ -104,10 +104,6 @@ protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { .addHint(ReadableQueryConfig.Hints.inexact); } - /** - * - * @param segment - */ @Override public void processSegment(SegmentContainer segment) { List features = this.getFeatures(segment); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java index dc3fc05a2..06224f07c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java @@ -120,7 +120,6 @@ protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { * Generates a set of feature vectors for the provided melody and returns them * * @param melody Melody for which to generate the vectors. - * @return */ private List getFeatures(Melody melody) { @@ -148,11 +147,6 @@ private List getFeatures(Melody melody) { return features; } - /** - * - * @param sc - * @return - */ private Melody transcribe(SegmentContainer sc) { /* Calculate STFT and apply spectral whitening. */ Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java index 1046d6806..6f819cb6d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java @@ -26,9 +26,6 @@ public abstract class SURF extends AbstractCodebookFeatureModule { private static QueryConfig.Distance DEFAULT_DISTANCE = QueryConfig.Distance.chisquared; - /** - * @param tableName - */ protected SURF(String tableName, int vectorLength) { super(tableName, 2.0f, vectorLength); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java index 4657c67f8..13296302a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java @@ -15,8 +15,6 @@ public SURFMirflickr25K256() { /** * Returns the full name of the Codebook to use. - * - * @return */ @Override protected String codebook() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java index c9b2a19d5..c41c06fde 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java @@ -15,8 +15,6 @@ public SURFMirflickr25K512() { /** * Returns the full name of the Codebook to use. - * - * @return */ @Override protected String codebook() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java index aca74d260..b9ba2eeb8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java @@ -30,10 +30,6 @@ public ShapeCentroidDistance() { super("features_shapecentroid", 2.0f, DESCRIPTOR_LENGTH); } - /** - * - * @param shot - */ @Override public void processSegment(SegmentContainer shot) { @@ -54,12 +50,6 @@ public void processSegment(SegmentContainer shot) { } } - /** - * - * @param sc - * @param qc - * @return - */ @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java index 62caa1a95..aa1858139 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java @@ -53,10 +53,6 @@ public SphericalHarmonics(String name, int grid_size, int min_l, int max_l) { this.voxelizer = new Voxelizer(2.0f/grid_size); } - /** - * - * @param shot - */ @Override public void processSegment(SegmentContainer shot) { /* Get the normalized Mesh. */ @@ -130,9 +126,6 @@ protected List postprocessQuery(List parti * * Depending on the model, the first components may be 0.0 because the surface of the sphere defined by the radius only * touches empty space (i.e the hollow interior of the model). - * - * @param mesh - * @return */ private float[] featureVectorFromMesh(ReadableMesh mesh) { final float increment = 0.1f; /* Increment of the angles during calculation of the descriptors. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java index f4310786a..9988bf9d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java @@ -33,19 +33,12 @@ public abstract class AbstractCodebookFeatureModule extends StagedFeatureModule /** The folder that contains the Codebook(s). */ private static String CODEBOOK_FOLDER = "resources/codebooks/"; - /** - * - * @param tableName - * @param maxDist - */ protected AbstractCodebookFeatureModule(String tableName, float maxDist, int vectorLength) { super(tableName, maxDist, vectorLength); } /** * Initializer for Extraction - must load the codebook. - * - * @param phandlerSupply */ @Override public final void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { @@ -56,9 +49,7 @@ public final void init(PersistencyWriterSupplier phandlerSupply, int batchSize) } /** - * Initializer for Retrieval - must load the codebook. - * - * @param selectorSupply + * Initializer for Retrieval - must load the codebook.selectorSupply */ @Override public final void init(DBSelectorSupplier selectorSupply) { @@ -129,8 +120,6 @@ protected final float[] floatToDoubleArray(double[] dbl) { /** * Returns the full name of the codebook to use. All codebook be placed in the * ./resources/codebooks folder. - * - * @return */ protected abstract String codebook(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java index 86d953b05..1ba899b32 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java @@ -40,11 +40,6 @@ public ImageCodebookGenerator(int vectorsize, boolean verbose) { this.cluster = new ClusterVisualWords(clusterer, vectorsize,0xA1CF3B12); } - /** - * @param source - * @param destination - * @param words - */ @Override public void generate(Path source, Path destination, int words) throws IOException { long start = System.currentTimeMillis(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java index 9efa29b07..366cdb4be 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java @@ -97,8 +97,6 @@ public static final int getAvg(Iterable colors){ } /** - * - * @param colors * @return value between 0 and 1 */ public static final float getAvgAlpha(Iterable colors){ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java index e9d84e18b..2f0c76168 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java @@ -34,8 +34,6 @@ public static MultiImage getEqualized(MultiImage in){ /** * * Equalizes the color histogram of the input image. - * @param in - * @return */ private static MultiImage equalizeHistogram(MultiImage in){ int[] inColors = in.getColors(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java index 6be6ac3a9..8288651cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java @@ -34,12 +34,9 @@ public static MotionHistoryImage motionHistoryImage(SegmentContainer container, return motionHistoryImage(container, lifeTime, threshold, true); } /** - * - * @param container * @param lifeTime number of frames to consider for image * @param threshold threshold distance [0, 255] * @param useThumbnails produce image based on thumbnails to entire frame - * @return */ public static MotionHistoryImage motionHistoryImage(SegmentContainer container, int lifeTime, int threshold, boolean useThumbnails){ if(container.getVideoFrames().isEmpty()){ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java index d969710ad..a5bc32dfe 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java @@ -38,13 +38,13 @@ public static Optional> and(Optional first, * If a value of the first {@code Optional} is present, returns it, otherwise returns an * {@code Optional} produced by the supplying function. * - * @param first the first {@code Optional} - * @param secondSupplier the supplying function that produces a second {@code Optional} to be + * @param first the first {@link Optional} + * @param secondSupplier the supplying function that produces a second {@link Optional} to be * returned - * @return an {@code Optional} describing the value of the first {@code Optional}, if present, - * otherwise an {@Optional} produced by the supplying function. - * @throws NullPointerException if the first {@code Optional} is null or the supplying function or - * its result is {@code null} + * @return an {@link Optional} describing the value of the first {@link Optional}, if present, + * otherwise an {@link Optional} produced by the supplying function. + * @throws NullPointerException if the first {@link Optional} is null or the supplying function or + * its result is null */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static Optional or(Optional first, Supplier> secondSupplier) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java index c2e1c8bef..60a76df58 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java @@ -271,7 +271,6 @@ private static Class[] getClassArray(Object... args) { * * @param cl The class that should be instantiated. * @param args The arguments that should be passed to the constructor. The constructor signature will be inferred from this list. - * @param * @return Instance of the class or null, if instantiation failed. */ public static T instantiate(Class cl, Object... args) { @@ -284,7 +283,6 @@ public static T instantiate(Class cl, Object... args) { * @param cl The class that should be instantiated. * @param types An array of types that defines the expected signature of the class's constructor. * @param args The arguments that should be passed to the constructor. - * @param * @return Instance of the class or null, if instantiation failed. */ public static T instantiate(Class cl, Class[] types, Object... args) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextStream.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextStream.java index 02f8f6bc7..23abd793e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextStream.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextStream.java @@ -22,8 +22,6 @@ public class TextStream { * @param first the index of the first frame * @param last the index of the last frame * @param coordinate_id the id of the coordinate - * @param coordinates1 - * @param coordinates2 */ public TextStream(int first, int last, int coordinate_id, List coordinates1, List coordinates2) { this.first = first; @@ -61,8 +59,6 @@ public void add(TextStream stream) { * @param start index of the first frame * @param end index of the last frame * @param coordinate_id id of the coordinate - * @param coordinates1 - * @param coordinates2 */ public void add(int start, int end, int coordinate_id, List coordinates1, List coordinates2) { for (int i = 0; i < coordinates1.size(); i++) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java index d3a87ee8f..54ba7f616 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java @@ -69,7 +69,6 @@ public static List calculate(STFT stft) { * @param cepstra The number of cepstra to obtain for the MFCC feature. * @param melfilters The number of triangular mel-filters (size of the mel-filter bank). * @param minFrequency Minimum frequency to consider for MFCC feature. - * @return */ public static List calculate(STFT stft, int cepstra, int melfilters, float minFrequency) { List spectra = stft.getMagnitudeSpectrum(); @@ -121,12 +120,6 @@ public static double melToFrequency(double mel) { return 700.0 * Math.pow(10, mel/2595.0) - 700.0; } - /** - * - * @param spectrum - * @param samplingrate - * @param windowsize - */ public void calculate(Spectrum spectrum, float samplingrate, int windowsize) { /* Check the type of the provided spectrum* */ if (spectrum.getType() != Spectrum.Type.MAGNITUDE) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java index b7144f0f4..aa10f6237 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java @@ -64,8 +64,6 @@ public float getFrequency() { /** * Getter for pitch salience. - * - * @return */ public double getSalience() { return salience; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java index 1d393b93d..3cf3153b6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java @@ -55,11 +55,6 @@ public KLF0PitchEstimator() { this(DEFAULT_MIN_PITCH, DEFAULT_MAX_PITCH, ALPHA, BETA); } - /** - * - * @param min - * @param max - */ public KLF0PitchEstimator(int min, int max, float alpha, float beta) { this.max = max; this.min = min; @@ -185,7 +180,7 @@ private Pitch detect(Spectrum spectrum, final float samplingrate, final int wind * @param f0 The f0 to calculate the salience for. * @param spectrum The spectrum to check. * @param samplingrate The rate at which the original audio has been sampled. - * @param samplingrate The windowsize used during creation of the spectrum. + * @param windowsize The windowsize used during creation of the spectrum. * @return Salience of f0 in the spectrum. */ private final double salience(float f0, Spectrum spectrum, final float samplingrate, final int windowsize) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java index 23caa4c77..1df7d53e5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java @@ -168,8 +168,6 @@ public final double salienceDeviation() { /** * Returns the sum of all salience values in the pitch contour. - * - * @return */ public final double salienceSum() { if (this.dirty) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java index d0afded7f..78fc3b698 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java @@ -98,8 +98,6 @@ public void initialize(List> candidates, float t_stepsize) { /** * Executes the pitch-tracking / pitch-streaming step - * - * @return */ public void trackPitches() { /* Apply the two filter stages described in [1], section II-B. */ @@ -138,10 +136,6 @@ public void trackPitches() { Collections.reverse( this.pitchContours); } - /** - * - * @return - */ public Melody extractMelody(int iterations) { /* Return if no pitch-contours are available. */ if (this.pitchContours.isEmpty()) { @@ -205,11 +199,6 @@ public Melody extractMelody(int iterations) { return melody; } - /** - * - * @param contours - * @return - */ public double[] meanContour(List contours) { final int size = 40; @@ -395,12 +384,6 @@ private void applyGlobalFilter() { } } - /** - * - * @param contour - * @param start - * @return - */ private void track(final PitchContour contour, final int start) { /* If start is the last entry, then no forward-tracking is required. */ if (start == this.s1.length - 1) { @@ -530,10 +513,6 @@ private void voicingDetection(List contours) { }); } - /** - * - * @param contours - */ private void detectAndRemoveOctaveDuplicates(List contours, double[] meanpitches) { Iterator iterator = contours.iterator(); while(iterator.hasNext()) { @@ -582,11 +561,6 @@ private void detectAndRemoveOctaveDuplicates(List contours, double } } - /** - * - * @param contours - * @param meanpitches - */ private void detectAndRemovePitchOutliers(List contours, final double[] meanpitches) { Iterator iterator = contours.iterator(); while(iterator.hasNext()) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java index bed878f1c..40cf5f2a4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java @@ -14,7 +14,6 @@ private SamplingUtilities() {} * * @param samples Double array that should be down-sampled. * @param factor Factor by which to down-sample. i.e. factor of n means that every n'th sample is discarded. - * @return */ public static double[] downsample (double[] samples, int factor) { /* Makes sure, that the factor is a positive value. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java index 4c61db71f..eb506abe9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java @@ -47,10 +47,8 @@ public static float binCenterFrequency(int index, int size, float samplingrate) * Returns the bin-index associated with the provided frequency at the given samplingrate * and window-size. * - * @param frequency * @param size Size of the FFT (i.e. number of frequency bins). * @param samplingrate Rate at which the original data has been sampled. - * @return */ public static int binIndex(float frequency, int size, float samplingrate) { if (frequency > samplingrate/2) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java index 91e723427..c47d01228 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java @@ -42,14 +42,6 @@ public class STFT { /** List containing one FFT entry per timepoint. Same order as time[] */ private final List stft; - /** - * - * @param windowsize - * @param overlap - * @param padding - * @param function - * @param samplingrate - */ public STFT(int windowsize, int overlap, int padding, WindowFunction function, float samplingrate) { /* Make sure that the windowsize is a power of two. */ if (!FFTUtil.isPowerOf2(windowsize)) { @@ -79,8 +71,6 @@ public STFT(int windowsize, int overlap, int padding, WindowFunction function, f * * Important: Every call to forward() appends a time-local DFT to the current STFT. The existing * data will be kept. - * - * @param samples */ public void forward(double[] samples) { /* Initialize values for the sliding window. */ @@ -150,8 +140,6 @@ public List getMagnitudeSpectrum() { /** * Getter for frequency bin labels. - * - * @return */ public final float[] getFrequencies() { return frequencies; @@ -166,8 +154,6 @@ public final int getNumberOfBins() { /** * Returns the size / width of an individual frequency bin. - * - * @return */ public final float getBinSize() { return (this.samplingrate/this.windowsize); @@ -175,8 +161,6 @@ public final float getBinSize() { /** * Getter for time labels. - * - * @return */ public final float[] getTime() { return time; @@ -184,8 +168,6 @@ public final float[] getTime() { /** * Getter for STFT. - * - * @return */ public final List getStft() { return Collections.unmodifiableList(this.stft); @@ -193,8 +175,6 @@ public final List getStft() { /** * Getter for window-size. - * - * @return */ public final int getWindowsize() { return this.windowsize; @@ -202,8 +182,6 @@ public final int getWindowsize() { /** * Getter for overlap value. - * - * @return */ public int getOverlap() { return this.overlap; @@ -211,8 +189,6 @@ public int getOverlap() { /** * Getter for sampling rate. - * - * @return */ public final float getSamplingrate() { return this.samplingrate; @@ -247,8 +223,6 @@ public int getHeight() { /** * The stepsize in seconds between to adjacent bins in the time dimension. - * - * @return */ public float timeStepsize() { return ((windowsize - overlap - 2*padding)/this.samplingrate); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java index 05fef6bda..24eb03565 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java @@ -82,12 +82,6 @@ public static Spectrum createMagnitudeSpectrum(Complex[] data, float samplingrat return new Spectrum(Type.MAGNITUDE, magnitudeSpectrum, FFTUtil.binCenterFrequencies(data.length, samplingrate)); } - /** - * - * @param type - * @param spectrum - * @param frequencies - */ public Spectrum(Type type, double[] spectrum, float[] frequencies) { /* Check if the length of both array is the same. */ if (spectrum.length != frequencies.length) { @@ -177,8 +171,6 @@ public void setValue(int idx, double value) { /** * Returns the maximum value in the spectrum. - * - * @return */ public Pair getMaximum() { if (this.maximumIndex == null) { @@ -194,8 +186,6 @@ public Pair getMaximum() { /** * Returns the minimum value in the spectrum. - * - * @return */ public Pair getMinimum() { if (this.minimumIndex == null) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/visualization/AudioSignalVisualizer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/visualization/AudioSignalVisualizer.java index c9125458c..fe0365baf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/visualization/AudioSignalVisualizer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/visualization/AudioSignalVisualizer.java @@ -148,7 +148,6 @@ public static BufferedImage visualizeCens(double[][] cens, int width, int height * @param min Minimal value in dB that should be color-coded. * @param max Maximal value in dB should be color-coded. * @param value value for which a color-code is required. - * @return */ public static Color color (double min, double max, double value) { if (min > max) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java index dc0dbec5e..5a96a7712 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java @@ -46,7 +46,6 @@ private SURFHelper() { * Returns SURF descriptors for an image using the settings above. Uses the BoofCV stable SURF algorithm. * * @param image Image for which to obtain the SURF descriptors. - * @return */ public static DetectDescribePoint getStableSurf(BufferedImage image) { /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ @@ -62,7 +61,6 @@ public static DetectDescribePoint getStableSurf(Buffered * which yields less images but operates a bit faster. * * @param image Image for which to obtain the SURF descriptors. - * @return */ public static DetectDescribePoint getFastSurf(BufferedImage image) { /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java index fea750036..f368fd0b5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java @@ -120,9 +120,6 @@ public static List zernikeMomentsForShapes(BufferedImage image, * Attempts at reconstructing an image from a list of complete Zernike Moments. The list must contain * a complete set of complex Zernike Moments up to some arbitrary order. The moments must be ordered according * to Noll's sequential index (ascending). - * - * @param moments - * @return */ public static BufferedImage reconstructImage(ZernikeMoments moments) { return ZernikeHelper.reconstructImage(moments.getHeight(), moments.getHeight(), moments.getMoments()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java index 239309d65..af8fae1c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java @@ -12,9 +12,6 @@ public interface JsonWriter { /** * Takes a Java Object (usually a POJO) and tries to serialize it into a JSON. If serialization * fails for some reason, this method should return JSON_EMPTY; - * - * @param object - * @return */ String toJson(Object object); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java index fddcfb212..5039b327f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java @@ -63,8 +63,6 @@ public Complex value(final double theta, final double phi) { * Calculates and returns the normalisation factor for the Spherical Harmonics Function * * @param l Order - * @param m - * @return */ public static double getFactor(int l, int m) { return FastMath.sqrt(((2*l + 1)/(4*Math.PI)) * ((double)CombinatoricsUtils.factorial(l-FastMath.abs(m)) / (double)CombinatoricsUtils.factorial(l+FastMath.abs(m)))); @@ -75,7 +73,6 @@ public static double getFactor(int l, int m) { * up to max_l. * * @param max_l The maximum harmonic to consider. - * @return */ public static int numberOfCoefficients(int max_l, boolean onesided) { int number = 0; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java index b04d3f346..573f8793a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java @@ -28,7 +28,6 @@ public class ZernikeBasisFunction implements UnivariateComplexFunction { * * @param n 1st moment of the Zernike Basis Function. * @param m 2nd moment of the Zernike Basis Function. - * @return ZernikeBasisFunction Z_nm * @throws IllegalArgumentException If |m| > n, m <'0 or (n-|m| % 2) = 1 */ public ZernikeBasisFunction(int n, int m) { @@ -79,9 +78,6 @@ public final PolynomialFunction getRadialPolynomial() { /** * Custom equals implementation. - * - * @param o - * @return */ @Override public boolean equals(Object o) { @@ -102,8 +98,6 @@ public boolean equals(Object o) { /** * Custom hashCode() implementation. - * - * @return */ @Override public int hashCode() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java index 18b4fbda0..3b9289bca 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java @@ -30,10 +30,6 @@ public static void color(WritableMesh mesh) { } } - /** - * - * @param mesh - */ public static void normalColoring(WritableMesh mesh) { Vector3f axis = new Vector3f(1.0f,0.0f, 0.0f); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java index 976a368f0..aa3ea2d57 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java @@ -38,12 +38,6 @@ public static BufferedImage dataURLtoBufferedImage(String dataUrl) { return bimg; } - /** - * - * @param img - * @param format - * @return - */ public static String BufferedImageToDataURL(BufferedImage img, String format){ ByteArrayOutputStream bouts = new ByteArrayOutputStream(); try { diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java index a2c0b3105..370365367 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java @@ -43,11 +43,6 @@ void testBlackmanHarrisWindow() { } - /** - * - * @param function - * @param length - */ private void executeTest(WindowFunction function, int length) { assertAll("Window Function Test (" + length +")", () -> testWindowFunctionZeroOutside(function, length), diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java index ece27988a..68e2f9835 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java @@ -68,8 +68,6 @@ public final void documentAvailable(String docID, int k, boolean relevant) { /** * Getter for query object ID. - * - * @return */ public final String getDocId() { return this.docID; @@ -77,8 +75,6 @@ public final String getDocId() { /** * Getter for query object class. - * - * @return */ public final String getCl() { return cl; @@ -114,17 +110,11 @@ public final int getIntersection() { /** * Returns true, if the number of retrieved & relevant documents equals the * total number of relevant documents. - * - * @return */ public boolean done() { return this.intersection == this.relevant; } - /** - * - * @return - */ public final String toString(String delimiter) { StringBuilder builder = new StringBuilder(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java index ae6a94c9c..b412e882c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java @@ -19,7 +19,6 @@ public class PlainTextImporter implements Importer> { * Constructor for {@link PlainTextImporter}. * * @param input Path to the input file or folder. - * @throws IOException */ public PlainTextImporter(Path input) throws IOException { this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java index d2a9028fc..adbc89a29 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java @@ -22,7 +22,6 @@ public class TagsFulltextImporter implements Importer> { * Constructor for {@link PlainTextImporter}. * * @param input Path to the input file or folder. - * @throws IOException */ public TagsFulltextImporter(Path input) throws IOException { this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java index deb8fb232..ee2afb329 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java @@ -23,7 +23,6 @@ public class VbsMetaImporter implements Importer> { * Constructor for {@link PlainTextImporter}. * * @param input Path to the input file or folder. - * @throws IOException */ public VbsMetaImporter(Path input) throws IOException { this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".xml")).iterator(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java index 8d19f9c0d..93995859b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java @@ -9,22 +9,12 @@ import java.nio.file.Path; public class AsrDataImportHandler extends DataImportHandler { - /** */ private static final Logger LOGGER = LogManager.getLogger(); - /** - * - * @param threads - * @param batchsize - */ public AsrDataImportHandler(int threads, int batchsize) { super(threads, batchsize); } - /** - * - * @param path - */ @Override public void doImport(Path path) { try { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java index 1c189ef7f..5111be2e5 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java @@ -13,11 +13,6 @@ public class JsonDataImportHandler extends DataImportHandler { /** */ private static final Logger LOGGER = LogManager.getLogger(); - /** - * - * @param threads - * @param batchsize - */ public JsonDataImportHandler(int threads, int batchsize) { super(threads, batchsize); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java index 7fb961c5f..82fe92399 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java @@ -14,19 +14,10 @@ public class OcrDataImportHandler extends DataImportHandler { /** */ private static final Logger LOGGER = LogManager.getLogger(); - /** - * - * @param threads - * @param batchsize - */ public OcrDataImportHandler(int threads, int batchsize) { super(threads, batchsize); } - /** - * - * @param path - */ @Override public void doImport(Path path) { try { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java index dbaf72a7c..1ccd99c97 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java @@ -13,11 +13,6 @@ public class ProtoDataImportHandler extends DataImportHandler { private static final Logger LOGGER = LogManager.getLogger(); - /** - * - * @param threads - * @param batchsize - */ public ProtoDataImportHandler(int threads, int batchsize) { super(threads, batchsize); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java index 8830b95ea..5ee575409 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java @@ -152,9 +152,6 @@ public static void writeLines(Path directory, String file, List lines) t * do contain more than the actual (image) file, hence some conversion is required *
    * Prepends is_, removes anything before a slash ("/"), if present, and after a dot (".") (i.e., file extension), if present - * - * @param path - * @return */ public static String pathToSegmentId(String path) { final int beginIdx = path.contains("/") ? path.lastIndexOf("/") + 1 : 0; @@ -198,9 +195,6 @@ public static Optional filenameToMinuteId(String filename) { /** * Removes all but after the last "/" - * - * @param path - * @return */ public static String sanitizeFilename(String path) { int i = path.lastIndexOf("/"); @@ -221,7 +215,6 @@ public static String sanitizeFilename(Path path) { * {@link LocalDateTime} is used, due to its definition of 'local time in ISO-8601 standard without time zone info'. * * @param lscUtcFormat The LSC UTC Timestamp in the format {@code UTC_yyyy-MM-dd_hh:mm} - * @return */ public static LocalDateTime convertUtc(String lscUtcFormat) { return LocalDateTime.ofEpochSecond(convert(lscUtcFormat, true, null), 0, ZoneOffset.UTC); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java index 1782a9377..92b70029c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java @@ -65,7 +65,6 @@ public Map readNext() { * Returns the next tag score element. * In other words, this returns the next triple segmentId, tagId, score als long as there are these triples. * Those triples are constructed by first getting the current segmentId and then iterating over this segment's tags until they are all processed - * @return */ private Map readNextTagScore() { do{ diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java index a34bf8862..674060d1a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java @@ -283,10 +283,7 @@ public Map readNext() { } /** - * Checks if the needle is in the unique set. if so, it's not unique. otherwise its unique and added to the list - * - * @param needle - * @return + * Checks if the needle is in the unique set. If so, it's not unique. Otherwise, it's unique and added to the list */ private boolean isUnique(String needle) { boolean found = uniqueList.contains(needle); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java index dd6e83106..f506a885a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java @@ -56,8 +56,6 @@ public ExtractionItemContainer( /** * To make fasterxml/jackson happy. - * - * @return */ public String getUri() { return uri; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java index b20b90b54..c3a02451a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java @@ -105,8 +105,6 @@ public synchronized boolean isRunning() { * @param container SegmentContainer to add to the queue. * @param timeout Time to wait for space to become available in ms. * @return true if SegmentContainer was emitted, false otherwise. - * - * @throws InterruptedException */ public boolean emit(SegmentContainer container, int timeout) throws InterruptedException { return this.segmentQueue.offer(container, timeout, TimeUnit.MILLISECONDS); From 8a2ff57912f55eb929b5ab26967e680037f30afc Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 16 Dec 2021 21:03:47 +0100 Subject: [PATCH 07/72] Removed unused commented code. --- .../cineast/api/rest/OpenApiCompatHelper.java | 2 -- .../cottontaildb/CottontailEntityCreator.java | 4 --- .../cineast/core/descriptor/AvgImg.java | 9 ++---- .../cineast/core/descriptor/MedianImg.java | 32 ------------------- .../importer/lsc2020/MetaImporter.java | 6 ---- 5 files changed, 2 insertions(+), 51 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java index dceddd721..c764427b3 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/OpenApiCompatHelper.java @@ -60,8 +60,6 @@ public static OpenApiOptions getJavalinOpenApiOptions(APIConfig config) { mapper.enable(SerializationFeature.INDENT_OUTPUT); mapper.addMixIn(Schema.class, SchemaMixin.class); // Makes Schema.exampleFlagSet being ignored by jackson -// mapper.addMixIn(MediaType.class, -// MediaTypeMixin.class); // Makes MediaType.exampleFlagSet being ignored by jackson return new OpenApiOptions(() -> getOpenApi(config)) .path("/openapi-specs") .activateAnnotationScanningFor("org.vitrivr.cineast.api") diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 59850dc73..3c7825875 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -324,10 +324,6 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { return Type.BOOL_VECTOR; case FLOAT: return Type.FLOAT; - /*case GEOGRAPHY: - return Type.GEOGRAPHY; - case GEOMETRY: - return Type.GEOMETRY;*/ case INT: return Type.INTEGER; case LONG: diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java index dd3374e98..2fd462aa3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java @@ -42,9 +42,7 @@ public static MultiImage getAvg(List videoFrames){ int size = videoFrames.size(); - - //BufferedImage _return = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); - + colors = new int[width * height]; for(int i = 0; i < colors.length; ++i){ @@ -53,10 +51,7 @@ public static MultiImage getAvg(List videoFrames){ (int)Math.round(buffer[3*i + 1] / size), (int)Math.round(buffer[3*i + 2] / size)); } - - //_return.setRGB(0, 0, width, height, colors, 0, width); - - //colors = null; + buffer = null; System.gc(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java index 868e3fadd..2c7a95027 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java @@ -101,38 +101,6 @@ public static MultiImage getMedian(List videoFrames){ System.gc(); LOGGER.traceExit(); return first.factory().newMultiImage(width, height, result); - - -// System.out.println("MedianImg.getMedian()"); -// -// MultiImage first = frames.get(0).getImage(); -// int width = first.getWidth(), height = first.getHeight(); -// FileCachedImageHistogram fcih = new FileCachedImageHistogram(width, height); -// for(Frame frame : frames){ -// BufferedImage bimg = frame.getImage().getBufferedImage(); -// for(int y = 0; y < height; ++y){ -// for(int x = 0; x < width; ++x){ -// int col = bimg.getRGB(x, y); -// fcih.updateBucket(x, y, 0, RGBContainer.getRed(col)); -// fcih.updateBucket(x, y, 1, RGBContainer.getGreen(col)); -// fcih.updateBucket(x, y, 2, RGBContainer.getBlue(col)); -// } -// System.err.println(y); -// } -// } -// -// BufferedImage median = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); -// for(int y = 0; y < height; ++y){ -// for(int x = 0; x < width; ++x){ -// int r = medianFromHistogram(fcih.getBucket(x, y, 0)); -// int g = medianFromHistogram(fcih.getBucket(x, y, 1)); -// int b = medianFromHistogram(fcih.getBucket(x, y, 2)); -// median.setRGB(x, y, RGBContainer.toIntColor(r, g, b)); -// } -// System.err.println(y); -// } -// -// return new MultiImage(median); } private static int medianFromHistogram(short[] hist){ diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java index 347cae3d2..cbc218a6b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java @@ -96,12 +96,6 @@ private Optional>> parseLine(String file } String minuteId = LSCUtilities.filenameToMinuteId(filename).get(); List> list = new ArrayList<>(); -// for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) { -// if(items[i].equalsIgnoreCase("null")){ -// continue; -// } -// list.add(parseMeta(filename, items, i)); -// } for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) { if (LSCUtilities.META_COLUMNS_IN_USE.contains(i)) { if (items[i].equalsIgnoreCase("null")) { From 0d9e9828c350c41fc73fdd1d8c5a902e7637dae0 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 16 Dec 2021 21:07:27 +0100 Subject: [PATCH 08/72] Fixed JavaDoc errors. --- .../cineast/core/extraction/decode/image/ImageSequence.java | 2 +- .../core/extraction/decode/image/ImageSequenceDecoder.java | 2 +- .../extraction/segmenter/image/ImageSequenceSegmenter.java | 2 +- .../standalone/monitoring/PrometheusExtractionTaskMonitor.java | 3 +-- .../cineast/standalone/run/ExtractionCompleteListener.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java index 7efda93d3..421b82ca4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java @@ -20,7 +20,7 @@ import java.util.function.Supplier; /** - * Represents a media object of type {@link MediaType.IMAGE_SEQUENCE}, i.e. a sequence of images contained in a folder. + * Represents a media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a folder. * This class is merely an internal abstraction of that type and the content it represents. Its sole purpose is to * provide lazy access to the images contained in such a sequence during the extraction process. * diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java index 8fc159b43..9597db52e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java @@ -17,7 +17,7 @@ import java.util.*; /** - * Decoder for media object of type {@link MediaType.IMAGE_SEQUENCE}, i.e. a sequence of images contained in a single + * Decoder for media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a single * folder that, in terms of Cineast's data model, belong together. * * Important: Unlike other implementations of {@link Decoder} this class operates on folders only! diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java index 2797b6d6e..24f980a7e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java @@ -40,7 +40,7 @@ public class ImageSequenceSegmenter implements Segmenter { /** * Instance of {@link FileNameObjectIdGenerator}. * - * Only Used in case that IDs should be derived from filenames which in the case of {@link MediaType.IMAGE_SEQUENCE} should also be done for the segments. + * Only Used in case that IDs should be derived from filenames which in the case of {@link MediaType#IMAGE_SEQUENCE} should also be done for the segments. */ private final FileNameObjectIdGenerator idgenerator; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java index 3149fc9cc..cd07a6b8a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java @@ -6,9 +6,8 @@ import org.vitrivr.cineast.standalone.config.Config; /** - * So we don't clutter the {@link org.vitrivr.cineast.standalone.runtime.ExtractionTask} code. + * So we don't clutter the ExtractionTask code. * Singleton where you can register an extraction time of features - * */ public class PrometheusExtractionTaskMonitor extends ImportTaskMonitor { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java index 8eda1c4b5..defca29b1 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java @@ -7,7 +7,7 @@ public interface ExtractionCompleteListener { /** * This method is called after the decoder for the object has been closed. - * There might still be scheduled or ongoing {@link org.vitrivr.cineast.standalone.runtime.ExtractionTask}s for this object. + * There might still be scheduled or ongoing ExtractionTasks for this object. */ default void onCompleted(ExtractionItemContainer path){ //ignore From 6a2c5cd414de5485f636e5477c6fe6883a2e2e55 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Thu, 16 Dec 2021 21:15:14 +0100 Subject: [PATCH 09/72] Raised source and target language compatibility to Java 9. This is required since FFMpegAudioDecoder and TechnicalVideoMetadataExtractor use Math.floorDiv, which is a function only available since Java 9. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a38a0ead3..1cd9b4402 100644 --- a/build.gradle +++ b/build.gradle @@ -36,8 +36,8 @@ subprojects { apply plugin: 'com.github.johnrengelman.shadow' - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 1.9 + targetCompatibility = 1.9 repositories { mavenCentral() From 4b2e8014f8afbf7565e3ba12e95dd375f207615b Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Fri, 17 Dec 2021 18:21:56 +0100 Subject: [PATCH 10/72] Raised source and target language compatibility to Java 11 (LTS). Due to popular demand. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1cd9b4402..615fb9c73 100644 --- a/build.gradle +++ b/build.gradle @@ -36,8 +36,8 @@ subprojects { apply plugin: 'com.github.johnrengelman.shadow' - sourceCompatibility = 1.9 - targetCompatibility = 1.9 + sourceCompatibility = 1.11 + targetCompatibility = 1.11 repositories { mavenCentral() From ef8d793e7ab9c7c4facfaa488bdc658d13f67a35 Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Wed, 22 Dec 2021 12:59:31 +0100 Subject: [PATCH 11/72] Reformatted all code to adhere to the project code-style. --- build.gradle | 2 +- cineast-api/build.gradle | 4 +- .../api/grpc/CineastExtractionService.java | 1 - .../api/grpc/CineastManagementService.java | 10 +- .../cineast/api/grpc/CineastQueryService.java | 581 ++++---- .../cineast/api/grpc/data/QueryStage.java | 29 +- .../cineast/api/grpc/data/QueryTerm.java | 59 +- .../api/grpc/util/MediaObjectUtil.java | 36 +- .../api/grpc/util/MediaSegmentUtil.java | 28 +- .../api/grpc/util/QueryContainerUtil.java | 519 +++---- .../query/NeighboringSegmentQuery.java | 89 +- .../api/messages/query/QueryComponent.java | 3 - .../api/messages/query/QueryTermType.java | 2 +- .../api/messages/query/SegmentQuery.java | 81 +- .../feature/FindTagsForElementGetHandler.java | 2 +- .../mediaobject/FindObjectGetHandler.java | 2 +- ...FindObjectMetadataByDomainPostHandler.java | 2 +- .../FindObjectMetadataByKeyPostHandler.java | 2 +- ...bjectMetadataFullyQualifiedGetHandler.java | 2 +- .../FindObjectMetadataGetHandler.java | 2 +- .../FindSegmentMetadataGetHandler.java | 2 +- .../segment/FindSegmentsByIdGetHandler.java | 2 +- .../FindSegmentsByObjectIdGetHandler.java | 2 +- .../actions/session/EndSessionHandler.java | 2 +- .../session/ValidateSessionHandler.java | 2 +- .../actions/tag/FindTagsGetHandler.java | 2 +- cineast-core/build.gradle | 4 +- .../cineast/core/color/ColorConverter.java | 4 +- .../color/FuzzyColorHistogramQuantizer.java | 496 +++---- .../cineast/core/color/HSVContainer.java | 55 +- .../cineast/core/color/LabContainer.java | 114 +- .../cineast/core/color/RGBContainer.java | 177 +-- .../core/color/ReadableHSVContainer.java | 12 +- .../core/color/ReadableLabContainer.java | 18 +- .../core/color/ReadableRGBContainer.java | 25 +- .../core/color/ReadableXYZContainer.java | 12 +- .../core/color/ReadableYCbCrContainer.java | 12 +- .../cineast/core/color/XYZContainer.java | 59 +- .../cineast/core/color/YCbCrContainer.java | 47 +- .../cineast/core/config/CacheConfig.java | 263 ++-- .../cineast/core/config/DatabaseConfig.java | 115 +- .../cineast/core/config/DecoderConfig.java | 148 +- .../vitrivr/cineast/core/config/IdConfig.java | 151 +-- .../cineast/core/config/QueryConfig.java | 4 +- .../core/config/ReadableQueryConfig.java | 327 ++--- .../cineast/core/config/SegmenterConfig.java | 144 +- .../core/data/CorrespondenceFunction.java | 2 +- .../core/data/DefaultValueHashMap.java | 14 +- .../vitrivr/cineast/core/data/DoublePair.java | 24 +- .../cineast/core/data/DynamicGrid.java | 31 +- .../cineast/core/data/ExistenceCheck.java | 4 +- .../core/data/FixedSizePriorityQueue.java | 5 +- .../cineast/core/data/FloatArrayIterable.java | 62 +- .../cineast/core/data/FloatVector.java | 8 +- .../cineast/core/data/FloatVectorImpl.java | 1 - .../vitrivr/cineast/core/data/GpsData.java | 141 +- .../org/vitrivr/cineast/core/data/Grid.java | 5 +- .../vitrivr/cineast/core/data/Histogram.java | 1 - .../cineast/core/data/InstantVector.java | 7 +- .../cineast/core/data/IntArrayIterable.java | 62 +- .../cineast/core/data/LimitedQueue.java | 49 +- .../vitrivr/cineast/core/data/Location.java | 15 +- .../vitrivr/cineast/core/data/MediaType.java | 386 +++--- .../org/vitrivr/cineast/core/data/Pair.java | 16 +- .../vitrivr/cineast/core/data/Position.java | 2 +- .../cineast/core/data/QuerySubTitleItem.java | 50 +- .../core/data/ReadableFloatVector.java | 5 +- .../cineast/core/data/SemanticMap.java | 5 +- .../cineast/core/data/StringDoublePair.java | 38 +- .../cineast/core/data/UniqueElementGrid.java | 9 +- .../distance/AbstractDistanceElement.java | 1 + .../core/data/distance/DistanceElement.java | 10 +- .../data/distance/ObjectDistanceElement.java | 1 + .../data/entities/MediaObjectDescriptor.java | 7 +- .../data/entities/MediaSegmentDescriptor.java | 249 ++-- .../entities/SimpleFeatureDescriptor.java | 40 +- .../core/data/entities/TagInstance.java | 2 +- .../core/data/frames/AudioDescriptor.java | 133 +- .../cineast/core/data/frames/AudioFrame.java | 538 ++++---- .../core/data/frames/VideoDescriptor.java | 125 +- .../cineast/core/data/frames/VideoFrame.java | 288 ++-- .../vitrivr/cineast/core/data/m3d/Mesh.java | 962 ++++++------- .../cineast/core/data/m3d/ReadableMesh.java | 114 +- .../cineast/core/data/m3d/VoxelGrid.java | 475 +++---- .../cineast/core/data/m3d/Voxelizer.java | 442 +++--- .../cineast/core/data/m3d/WritableMesh.java | 94 +- .../data/providers/AudioFrameProvider.java | 216 ++- .../data/providers/AudioSTFTProvider.java | 60 +- .../core/data/providers/AvgImgProvider.java | 9 +- .../providers/BooleanExpressionProvider.java | 5 +- .../core/data/providers/DurationProvider.java | 75 +- .../data/providers/FrameListProvider.java | 14 +- .../core/data/providers/IdProvider.java | 23 +- .../core/data/providers/InstantProvider.java | 1 + .../core/data/providers/LocationProvider.java | 4 +- .../data/providers/MedianImgProvider.java | 9 +- .../core/data/providers/MeshProvider.java | 81 +- .../MostRepresentativeFrameProvider.java | 11 +- .../core/data/providers/PathProvider.java | 19 +- .../core/data/providers/SegmentProvider.java | 7 +- .../data/providers/SemanticMapProvider.java | 9 +- .../data/providers/SubtitleItemProvider.java | 11 +- .../core/data/providers/TagProvider.java | 11 +- .../core/data/providers/TextProvider.java | 4 +- .../data/providers/VoxelGridProvider.java | 18 +- .../providers/primitive/BitSetProvider.java | 1 + .../primitive/BitSetTypeProvider.java | 11 +- .../providers/primitive/BooleanProvider.java | 10 +- .../primitive/BooleanProviderImpl.java | 60 +- .../primitive/BooleanTypeProvider.java | 22 +- .../providers/primitive/ByteProvider.java | 8 +- .../providers/primitive/ByteProviderImpl.java | 64 +- .../providers/primitive/ByteTypeProvider.java | 2 +- .../providers/primitive/DoubleProvider.java | 10 +- .../primitive/DoubleProviderImpl.java | 82 +- .../primitive/DoubleTypeProvider.java | 22 +- .../primitive/FloatArrayProvider.java | 10 +- .../primitive/FloatArrayProviderImpl.java | 74 +- .../primitive/FloatArrayTypeProvider.java | 8 +- .../providers/primitive/FloatProvider.java | 10 +- .../primitive/FloatProviderImpl.java | 68 +- .../primitive/FloatTypeProvider.java | 22 +- .../primitive/FloatVectorProvider.java | 18 +- .../providers/primitive/IntArrayProvider.java | 12 +- .../primitive/IntArrayProviderImpl.java | 66 +- .../primitive/IntArrayTypeProvider.java | 5 +- .../data/providers/primitive/IntProvider.java | 10 +- .../providers/primitive/IntProviderImpl.java | 88 +- .../providers/primitive/IntTypeProvider.java | 22 +- .../primitive/IntVectorProvider.java | 22 +- .../providers/primitive/LongProvider.java | 10 +- .../providers/primitive/LongProviderImpl.java | 76 +- .../providers/primitive/LongTypeProvider.java | 22 +- .../providers/primitive/NothingProvider.java | 14 +- .../PrimitiveProviderComparator.java | 14 +- .../primitive/PrimitiveTypeProvider.java | 1 - .../providers/primitive/ProviderDataType.java | 4 +- .../providers/primitive/ShortProvider.java | 10 +- .../primitive/ShortProviderImpl.java | 64 +- .../primitive/ShortTypeProvider.java | 4 +- .../providers/primitive/StringProvider.java | 10 +- .../primitive/StringProviderImpl.java | 68 +- .../primitive/StringTypeProvider.java | 18 +- .../AbstractQueryTermContainer.java | 6 +- .../containers/AudioQueryTermContainer.java | 151 ++- .../containers/BooleanQueryTermContainer.java | 11 +- .../containers/IdQueryTermContainer.java | 6 +- .../containers/ImageQueryTermContainer.java | 231 ++-- .../LocationQueryTermContainer.java | 83 +- .../containers/ModelQueryTermContainer.java | 184 +-- .../containers/MotionQueryTermContainer.java | 139 +- ...rameterisedLocationQueryTermContainer.java | 4 +- .../SemanticMapQueryTermContainer.java | 114 +- .../containers/TextQueryTermContainer.java | 32 +- .../cineast/core/data/raw/CacheableData.java | 15 +- .../core/data/raw/CachedDataFactory.java | 573 ++++---- .../cineast/core/data/raw/bytes/ByteData.java | 49 +- .../core/data/raw/bytes/CachedByteData.java | 271 ++-- .../core/data/raw/bytes/InMemoryByteData.java | 120 +- .../data/raw/images/CachedMultiImage.java | 402 +++--- .../data/raw/images/InMemoryMultiImage.java | 270 ++-- .../core/data/raw/images/MultiImage.java | 61 +- .../core/data/score/AbstractScoreElement.java | 1 + .../score/BooleanSegmentScoreElement.java | 2 +- .../core/data/score/ObjectScoreElement.java | 1 + .../cineast/core/data/score/ScoreElement.java | 3 - .../core/data/score/SegmentScoreElement.java | 7 +- .../core/data/segments/AudioSegment.java | 347 ++--- .../core/data/segments/ImageSegment.java | 159 ++- .../core/data/segments/Model3DSegment.java | 179 +-- .../core/data/segments/SegmentContainer.java | 57 +- .../core/data/segments/VideoSegment.java | 710 +++++----- .../cineast/core/data/tag/CompleteTag.java | 4 +- .../core/data/tag/CompleteWeightedTag.java | 4 +- .../cineast/core/data/tag/IncompleteTag.java | 4 +- .../cineast/core/data/tag/TagPriority.java | 6 +- .../core/db/AbstractPersistencyWriter.java | 18 +- .../cineast/core/db/BooleanExpression.java | 7 +- .../cineast/core/db/DBSelectorSupplier.java | 2 +- .../cineast/core/db/DataMessageConverter.java | 429 +++--- .../cineast/core/db/MergeOperation.java | 2 +- .../vitrivr/cineast/core/db/NoDBSelector.java | 92 +- .../cineast/core/db/PersistentOperator.java | 3 +- .../cineast/core/db/PersistentTuple.java | 36 +- .../cineast/core/db/RelationalOperator.java | 74 +- .../core/db/adampro/ADAMproEntityCreator.java | 493 ++++--- .../db/adampro/ADAMproMessageBuilder.java | 756 ++++++----- .../core/db/adampro/ADAMproSelector.java | 542 ++++---- .../db/adampro/ADAMproStreamingSelector.java | 55 +- .../core/db/adampro/ADAMproWrapper.java | 41 +- .../core/db/adampro/ADAMproWriter.java | 17 +- .../db/adampro/AbstractADAMproSelector.java | 23 +- .../db/adampro/ProtobufTupleGenerator.java | 167 ++- .../cottontaildb/CottontailEntityCreator.java | 29 +- .../db/cottontaildb/CottontailWrapper.java | 9 +- .../db/cottontaildb/CottontailWriter.java | 1 - .../db/dao/MetadataAccessSpecification.java | 22 +- .../db/dao/reader/AbstractEntityReader.java | 43 +- .../db/dao/reader/AbstractMetadataReader.java | 218 ++- .../core/db/dao/reader/MediaObjectReader.java | 1 - .../db/dao/reader/MediaSegmentReader.java | 26 +- .../writer/AbstractBatchedEntityWriter.java | 153 ++- .../core/db/dao/writer/BatchedTagWriter.java | 4 +- .../core/db/dao/writer/MediaObjectWriter.java | 32 +- .../writer/SimpleFeatureDescriptorWriter.java | 44 +- .../cineast/core/db/dao/writer/TagWriter.java | 75 +- .../cineast/core/db/json/JsonFileWriter.java | 43 +- .../cineast/core/db/json/JsonSelector.java | 10 +- .../cineast/core/db/memory/InMemoryStore.java | 58 +- .../core/db/memory/InMemoryWriter.java | 1 - .../core/db/setup/AttributeDefinition.java | 198 +-- .../core/db/setup/EntityDefinition.java | 322 ++--- .../core/db/setup/NoEntityCreator.java | 4 +- .../cineast/core/descriptor/AvgImg.java | 101 +- .../cineast/core/descriptor/EdgeImg.java | 206 +-- .../cineast/core/descriptor/EdgeList.java | 88 +- .../core/descriptor/ImageDistance.java | 73 +- .../cineast/core/descriptor/MedianImg.java | 217 ++- .../core/descriptor/MostRepresentative.java | 41 +- .../cineast/core/descriptor/PathList.java | 393 +++--- .../extraction/ExtractionContextProvider.java | 1 - .../extraction/decode/audio/AudioDecoder.java | 4 +- .../decode/audio/FFMpegAudioDecoder.java | 762 ++++++----- .../extraction/decode/general/Converter.java | 30 +- .../extraction/decode/general/Decoder.java | 123 +- .../decode/image/DefaultImageDecoder.java | 274 ++-- .../decode/image/ImageSequence.java | 164 +-- .../decode/image/ImageSequenceDecoder.java | 58 +- .../extraction/decode/m3d/MeshDecoder.java | 4 +- .../decode/m3d/MeshDecoderException.java | 6 +- .../decode/m3d/ModularMeshDecoder.java | 360 ++--- .../extraction/decode/m3d/OBJMeshDecoder.java | 290 ++-- .../extraction/decode/m3d/OFFMeshDecoder.java | 375 ++--- .../extraction/decode/m3d/STLMeshDecoder.java | 518 +++---- .../shotboundary/ShotBoundaryDecoder.java | 84 +- .../TrecvidMasterShotReferenceDecoder.java | 131 +- .../decode/subtitle/AbstractSubtitleItem.java | 69 +- .../decode/subtitle/SubTitleDecoder.java | 67 +- .../subtitle/SubtitleDecoderFactory.java | 137 +- .../decode/subtitle/SubtitleItem.java | 51 +- .../decode/subtitle/cc/CCSubTitleDecoder.java | 281 ++-- .../decode/subtitle/cc/CCSubTitleItem.java | 16 +- .../subtitle/srt/SRTSubTitleDecoder.java | 265 ++-- .../decode/subtitle/srt/SRTSubtitleItem.java | 28 +- .../video/AbstractAVStreamContainer.java | 104 +- .../video/AudioOutputStreamContainer.java | 340 ++--- .../decode/video/FFMpegVideoDecoder.java | 1207 +++++++++-------- .../decode/video/FFMpegVideoEncoder.java | 238 ++-- .../video/VideoOutputStreamContainer.java | 296 ++-- .../FileNameObjectIdGenerator.java | 5 +- .../JSONProvidedObjectIdGenerator.java | 129 +- .../idgenerator/ObjectIdGenerator.java | 36 +- .../SequentialObjectIdGenerator.java | 89 +- .../idgenerator/Sha1ObjectIdGenerator.java | 9 +- .../idgenerator/UniqueObjectIdGenerator.java | 97 +- .../metadata/EXIFMetadataExtractor.java | 19 +- .../metadata/IIIFMetaDataExtractor.java | 1 - .../metadata/IPTCMetadataExtractor.java | 45 +- .../metadata/JsonMetaDataExtractor.java | 33 +- .../metadata/MetadataExtractor.java | 15 +- .../metadata/MetadataFeatureExtractor.java | 16 +- .../TechnicalVideoMetadataExtractor.java | 25 +- .../segmenter/FuzzyColorHistogram.java | 42 +- .../FuzzyColorHistogramCalculator.java | 134 +- .../SubdividedFuzzyColorHistogram.java | 54 +- .../audio/ConstantLengthAudioSegmenter.java | 367 ++--- .../general/PassthroughSegmenter.java | 10 +- .../segmenter/general/Segmenter.java | 65 +- .../segmenter/image/ImageSegmenter.java | 71 +- .../image/ImageSequenceSegmenter.java | 49 +- .../segmenter/video/TRECVidMSRSegmenter.java | 447 +++--- .../segmenter/video/V3CMSBSegmenter.java | 61 +- .../video/VideoHistogramSegmenter.java | 427 +++--- .../core/features/AudioFingerprint.java | 355 +++-- .../cineast/core/features/AverageColor.java | 5 +- .../core/features/AverageColorARP44.java | 5 +- .../features/AverageColorARP44Normalized.java | 5 +- .../core/features/AverageColorCLD.java | 5 +- .../features/AverageColorCLDNormalized.java | 5 +- .../core/features/AverageColorGrid8.java | 12 +- .../features/AverageColorGrid8Normalized.java | 5 +- .../features/AverageColorGrid8Reduced11.java | 5 +- .../features/AverageColorGrid8Reduced15.java | 5 +- .../core/features/AverageColorRaster.java | 245 ++-- .../features/AverageColorRasterReduced11.java | 43 +- .../features/AverageColorRasterReduced15.java | 43 +- .../core/features/AverageFuzzyHist.java | 5 +- .../features/AverageFuzzyHistNormalized.java | 5 +- .../cineast/core/features/AverageHPCP.java | 249 ++-- .../core/features/AverageHPCP20F36B.java | 17 +- .../core/features/AverageHPCP30F36B.java | 17 +- .../vitrivr/cineast/core/features/CENS.java | 339 +++-- .../core/features/CENS12BasslineShingle.java | 30 +- .../core/features/CENS12MelodyShingle.java | 30 +- .../cineast/core/features/CENS12Shingle.java | 27 +- .../vitrivr/cineast/core/features/CLD.java | 3 +- .../cineast/core/features/CLDNormalized.java | 3 +- .../cineast/core/features/CLDReduced11.java | 3 +- .../cineast/core/features/CLDReduced15.java | 3 +- .../cineast/core/features/ChromaGrid8.java | 14 +- .../cineast/core/features/ConceptMasks.java | 15 +- .../core/features/ConceptMasksAde20k.java | 10 +- .../cineast/core/features/DCTImageHash.java | 12 +- .../DailyCollectionBooleanRetriever.java | 10 +- .../features/DailyRangeBooleanRetriever.java | 7 +- .../cineast/core/features/DominantColors.java | 7 +- .../core/features/DominantEdgeGrid16.java | 7 +- .../core/features/DominantEdgeGrid8.java | 7 +- .../vitrivr/cineast/core/features/EHD.java | 11 +- .../cineast/core/features/EdgeARP88.java | 9 +- .../cineast/core/features/EdgeARP88Full.java | 9 +- .../cineast/core/features/EdgeGrid16.java | 9 +- .../cineast/core/features/EdgeGrid16Full.java | 9 +- .../core/features/ForegroundBoundingBox.java | 7 +- .../vitrivr/cineast/core/features/HOG.java | 9 +- .../core/features/HOGMirflickr25K256.java | 32 +- .../core/features/HOGMirflickr25K512.java | 32 +- .../core/features/HPCP12BasslineShingle.java | 30 +- .../core/features/HPCP12MelodyShingle.java | 30 +- .../cineast/core/features/HPCP12Shingle.java | 17 +- .../cineast/core/features/HPCPShingle.java | 360 +++-- .../cineast/core/features/HueHistogram.java | 17 +- .../core/features/HueValueVarianceGrid8.java | 14 +- .../cineast/core/features/Lightfield.java | 374 +++-- .../core/features/LightfieldFourier.java | 175 ++- .../core/features/LightfieldZernike.java | 151 +-- .../cineast/core/features/MFCCShingle.java | 236 ++-- .../cineast/core/features/MedianColor.java | 5 +- .../core/features/MedianColorARP44.java | 5 +- .../features/MedianColorARP44Normalized.java | 5 +- .../core/features/MedianColorGrid8.java | 12 +- .../features/MedianColorGrid8Normalized.java | 5 +- .../core/features/MedianColorRaster.java | 37 +- .../core/features/MedianFuzzyHist.java | 3 +- .../features/MedianFuzzyHistNormalized.java | 3 +- .../cineast/core/features/MelodyEstimate.java | 261 ++-- .../core/features/MotionHistogram.java | 7 +- .../features/MotionHistogramBackground.java | 7 +- .../cineast/core/features/OCRSearch.java | 55 +- .../core/features/ObjectInstances.java | 3 +- .../core/features/RangeBooleanRetriever.java | 28 +- .../cineast/core/features/STMP7EH.java | 7 +- .../vitrivr/cineast/core/features/SURF.java | 133 +- .../core/features/SURFMirflickr25K256.java | 31 +- .../core/features/SURFMirflickr25K512.java | 31 +- .../core/features/SaturationGrid8.java | 14 +- .../core/features/ShapeCentroidDistance.java | 99 +- .../core/features/SpatialDistance.java | 15 +- .../core/features/SphericalHarmonics.java | 312 +++-- .../features/SphericalHarmonicsDefault.java | 25 +- .../core/features/SphericalHarmonicsHigh.java | 25 +- .../core/features/SphericalHarmonicsLow.java | 25 +- .../features/SubDivAverageFuzzyColor.java | 3 +- .../core/features/SubDivMedianFuzzyColor.java | 3 +- .../core/features/SubDivMotionHistogram2.java | 5 +- .../core/features/SubDivMotionHistogram3.java | 5 +- .../core/features/SubDivMotionHistogram4.java | 5 +- .../core/features/SubDivMotionHistogram5.java | 5 +- .../SubDivMotionHistogramBackground2.java | 5 +- .../SubDivMotionHistogramBackground3.java | 75 +- .../SubDivMotionHistogramBackground4.java | 77 +- .../SubDivMotionHistogramBackground5.java | 5 +- .../core/features/SubDivMotionSum2.java | 5 +- .../core/features/SubDivMotionSum3.java | 5 +- .../core/features/SubDivMotionSum4.java | 5 +- .../core/features/SubDivMotionSum5.java | 5 +- .../features/SubDivMotionSumBackground2.java | 5 +- .../features/SubDivMotionSumBackground3.java | 5 +- .../features/SubDivMotionSumBackground4.java | 5 +- .../features/SubDivMotionSumBackground5.java | 23 +- .../core/features/TemporalDistance.java | 24 +- .../WSDMTICollectionBooleanRetriever.java | 10 +- .../AbstractCodebookFeatureModule.java | 204 ++- .../abstracts/AbstractFeatureModule.java | 274 ++-- .../abstracts/AbstractTextRetriever.java | 4 +- .../features/abstracts/BooleanRetriever.java | 42 +- .../abstracts/MetadataFeatureModule.java | 38 +- .../abstracts/MotionHistogramCalculator.java | 11 +- .../abstracts/StagedFeatureModule.java | 323 +++-- .../abstracts/SubDivMotionHistogram.java | 6 +- .../features/codebook/CodebookGenerator.java | 20 +- .../codebook/HOGCodebookGenerator.java | 53 +- .../codebook/ImageCodebookGenerator.java | 234 ++-- .../codebook/SURFCodebookGenerator.java | 53 +- .../exporter/AudioSegmentExporter.java | 232 ++-- .../exporter/AudioSpectogramExporter.java | 187 +-- .../exporter/AudioWaveformExporter.java | 264 ++-- .../core/features/exporter/CENSExporter.java | 173 +-- .../features/exporter/ChromagramExporter.java | 162 +-- .../core/features/exporter/FrameExporter.java | 124 +- .../exporter/Model3DThumbnailExporter.java | 233 ++-- .../exporter/MotionFrameExporter.java | 212 +-- .../exporter/MotionHistoryImageExporter.java | 132 +- .../features/exporter/QueryImageExporter.java | 140 +- .../exporter/RepresentativeFrameExporter.java | 133 +- .../exporter/ShotDescriptorExporter.java | 158 +-- .../exporter/ShotThumbnailsExporter.java | 164 +-- .../core/features/extractor/Extractor.java | 9 +- .../core/features/neuralnet/ImageCropper.java | 25 +- .../features/neuralnet/tf/GraphHelper.java | 14 +- .../neuralnet/tf/models/Inception5h.java | 15 +- .../neuralnet/tf/models/deeplab/DeepLab.java | 14 +- .../tf/models/deeplab/DeepLabLabel.java | 9 +- .../neuralnet/tf/models/yolo/YOLO.java | 24 +- .../tf/models/yolo/util/BoundingBox.java | 109 +- .../tf/models/yolo/util/BoxPosition.java | 217 +-- .../tf/models/yolo/util/Recognition.java | 100 +- .../neuralnet/tf/models/yolo/util/Size.java | 4 +- .../retriever/CollectionBooleanRetriever.java | 7 +- .../MultipleInstantiatableRetriever.java | 10 +- .../core/features/retriever/Retriever.java | 33 +- .../retriever/RetrieverInitializer.java | 4 +- .../vitrivr/cineast/core/iiif/IIIFItem.java | 4 +- .../imageapi/BaseImageRequestValidators.java | 8 +- .../core/iiif/imageapi/ImageFactory.java | 4 +- .../core/iiif/imageapi/ImageMetadata.java | 2 +- .../core/iiif/imageapi/ImageRequest.java | 4 +- .../v2/ImageInformationRequest_v2.java | 4 +- .../v3/ImageInformationRequest_v3.java | 4 +- .../presentationapi/v2/ManifestFactory.java | 2 +- .../cineast/core/importer/Importer.java | 91 +- .../core/importer/JsonObjectImporter.java | 9 +- .../importer/TupleInsertMessageImporter.java | 87 +- .../core/render/JOGLOffscreenRenderer.java | 818 +++++------ .../vitrivr/cineast/core/render/Renderer.java | 359 +++-- .../cineast/core/util/ARPartioner.java | 177 ++- .../core/util/ColorLayoutDescriptor.java | 160 +-- .../cineast/core/util/ColorReductionUtil.java | 41 +- .../vitrivr/cineast/core/util/ColorUtils.java | 247 ++-- .../cineast/core/util/DecodingError.java | 4 +- .../cineast/core/util/GridPartitioner.java | 37 +- .../cineast/core/util/GroupingUtil.java | 19 +- .../cineast/core/util/HungarianAlgorithm.java | 55 +- .../core/util/ImageHistogramEqualizer.java | 161 ++- .../vitrivr/cineast/core/util/KMeansPP.java | 11 +- .../vitrivr/cineast/core/util/LogHelper.java | 23 +- .../cineast/core/util/MaskGenerator.java | 722 +++++----- .../vitrivr/cineast/core/util/MathHelper.java | 535 ++++---- .../cineast/core/util/MetadataUtil.java | 36 +- .../cineast/core/util/MimeTypeHelper.java | 15 +- .../cineast/core/util/MotionHistoryImage.java | 178 +-- .../cineast/core/util/MultiTracker.java | 7 +- .../cineast/core/util/OptionalUtil.java | 44 +- .../cineast/core/util/QueryIDGenerator.java | 2 +- .../core/util/RandomStringGenerator.java | 23 +- .../cineast/core/util/ReflectionHelper.java | 545 ++++---- .../cineast/core/util/ScoreFusion.java | 19 +- .../cineast/core/util/TextDetector_EAST.java | 24 +- .../cineast/core/util/TextRecognizer_CTC.java | 14 +- .../vitrivr/cineast/core/util/TextStream.java | 20 +- .../core/util/ThreadLocalObjectCache.java | 21 +- .../vitrivr/cineast/core/util/TimeHelper.java | 50 +- .../vitrivr/cineast/core/util/audio/CENS.java | 97 +- .../vitrivr/cineast/core/util/audio/HPCP.java | 422 +++--- .../vitrivr/cineast/core/util/audio/MFCC.java | 358 +++-- .../cineast/core/util/audio/pitch/Melody.java | 189 +-- .../cineast/core/util/audio/pitch/Pitch.java | 243 ++-- .../pitch/estimation/KLF0PitchEstimator.java | 398 +++--- .../audio/pitch/tracking/PitchContour.java | 390 +++--- .../audio/pitch/tracking/PitchTracker.java | 1038 +++++++------- .../core/util/distance/BitSetComparator.java | 3 +- .../util/distance/BitSetHammingDistance.java | 2 +- .../distance/BoundedManhattanDistance.java | 2 +- .../util/distance/FloatArrayDistance.java | 112 +- .../core/util/distance/HaversineDistance.java | 4 +- .../PrimitiveTypeMapDistanceComparator.java | 3 +- .../distance/WeightedChebyshevDistance.java | 20 +- .../distance/WeightedManhattanDistance.java | 2 +- .../distance/WeightedMinkowskiDistance.java | 4 +- .../distance/WeightedSpanNormDistance.java | 4 +- .../WeightedSquaredEuclideanDistance.java | 20 +- .../cineast/core/util/dsp/FrequencyUtils.java | 34 +- .../core/util/dsp/SamplingUtilities.java | 51 +- .../cineast/core/util/dsp/fft/FFT.java | 269 ++-- .../cineast/core/util/dsp/fft/FFTUtil.java | 212 ++- .../cineast/core/util/dsp/fft/STFT.java | 471 +++---- .../cineast/core/util/dsp/fft/Spectrum.java | 575 ++++---- .../dsp/fft/windows/BlackmanHarrisWindow.java | 40 +- .../util/dsp/fft/windows/HanningWindow.java | 62 +- .../dsp/fft/windows/RectangularWindow.java | 50 +- .../util/dsp/fft/windows/WindowFunction.java | 66 +- .../FrequencyDomainFilterInterface.java | 35 +- .../frequency/SpectralWhiteningFilter.java | 261 ++-- .../spectrum/SpectralWhiteningFilter.java | 148 +- .../spectrum/SpectrumFilterInterface.java | 15 +- .../time/TimeDomainFilterInterface.java | 38 +- .../cineast/core/util/dsp/midi/MidiUtil.java | 112 +- .../visualization/AudioSignalVisualizer.java | 281 ++-- .../core/util/images/ContourHelper.java | 419 +++--- .../cineast/core/util/images/HOGHelper.java | 110 +- .../cineast/core/util/images/SURFHelper.java | 103 +- .../core/util/images/ZernikeHelper.java | 282 ++-- .../core/util/json/JacksonJsonProvider.java | 22 +- .../cineast/core/util/json/JsonReader.java | 21 +- .../cineast/core/util/json/JsonWriter.java | 15 +- .../cineast/core/util/math/MathConstants.java | 65 +- .../core/util/math/ZernikeMoments.java | 224 +-- .../AssociatedLegendrePolynomial.java | 141 +- .../functions/SphericalHarmonicsFunction.java | 163 +-- .../math/functions/ZernikeBasisFunction.java | 167 +-- .../factories/PolynomialFunctionFactory.java | 91 +- .../interfaces/UnivariateComplexFunction.java | 19 +- .../core/util/mesh/MeshColoringUtil.java | 65 +- .../cineast/core/util/mesh/MeshMathUtil.java | 246 ++-- .../core/util/mesh/MeshTransformUtil.java | 372 +++-- .../cineast/core/util/web/AudioParser.java | 137 +- .../cineast/core/util/web/DataURLParser.java | 230 ++-- .../cineast/core/util/web/ImageParser.java | 95 +- .../cineast/core/util/web/MeshParser.java | 120 +- .../cineast/core/data/LocationTest.java | 14 +- .../cineast/core/db/DBIntegrationTest.java | 36 +- .../cottontaildb/CottontailMetadataTest.java | 1 - .../cineast/core/db/dao/MetadataTest.java | 2 +- .../core/temporal/TemporalTestCases.java | 2 +- ...equentialTemporalScoringAlgorithmTest.java | 1 - ...eDistanceTemporalScoringAlgorithmTest.java | 1 + .../core/util/dsp/fft/WindowFunctionTest.java | 150 +- .../core/util/dsp/midi/MidiUtilTest.java | 60 +- .../cineast/core/util/math/MeshMathTest.java | 166 ++- .../AssociatedLegendrePolynomialTest.java | 352 +++-- .../math/functions/RadialPolynomialTest.java | 181 +-- .../functions/ZernikePolynomialsTest.java | 137 +- cineast-runtime/build.gradle | 4 +- .../org/vitrivr/cineast/standalone/Main.java | 4 +- .../cineast/standalone/cli/CliUtils.java | 2 +- .../standalone/cli/CodebookCommand.java | 1 - .../standalone/cli/DatabaseSetupCommand.java | 6 +- .../standalone/cli/ExtractionCommand.java | 2 +- .../cineast/standalone/cli/ImportCommand.java | 5 +- .../cli/OptimizeEntitiesCommand.java | 1 - .../standalone/cli/RetrieveCommand.java | 2 - .../cli/db/LSC21TemporalUpdateCommand.java | 2 - .../cineast/standalone/config/APIConfig.java | 1 - .../cineast/standalone/config/Config.java | 179 +-- .../config/ConstrainedQueryConfig.java | 42 +- .../standalone/config/ExplorativeConfig.java | 97 +- .../config/ExtractionPipelineConfig.java | 127 +- .../standalone/config/ExtractorConfig.java | 82 +- .../standalone/config/IngestConfig.java | 651 ++++----- .../standalone/config/MetadataConfig.java | 74 +- .../standalone/config/MonitoringConfig.java | 4 +- .../config/RetrievalRuntimeConfig.java | 333 +++-- .../standalone/config/RetrieverConfig.java | 83 +- .../evaluation/EvaluationConfig.java | 225 +-- .../evaluation/EvaluationException.java | 19 +- .../evaluation/EvaluationResult.java | 290 ++-- .../evaluation/EvaluationRuntime.java | 392 +++--- .../standalone/evaluation/Groundtruth.java | 233 ++-- .../cineast/standalone/importer/Copier.java | 187 ++- .../standalone/importer/LIREImporter.java | 14 +- .../importer/PlainTextImporter.java | 96 +- .../importer/TagsFulltextImporter.java | 96 +- .../standalone/importer/VbsMetaImporter.java | 105 +- .../handlers/AsrDataImportHandler.java | 28 +- .../importer/handlers/DataImportHandler.java | 349 +++-- .../handlers/JsonDataImportHandler.java | 62 +- .../importer/handlers/LIREImportHandler.java | 63 +- .../handlers/OcrDataImportHandler.java | 9 +- .../handlers/ProtoDataImportHandler.java | 7 +- .../lsc2020/CaptionImportHandler.java | 33 +- .../importer/lsc2020/CaptionImporter.java | 89 +- .../importer/lsc2020/JsonTagReader.java | 25 +- .../importer/lsc2020/LSCUtilities.java | 852 ++++++------ .../importer/lsc2020/MetaImportHandler.java | 57 +- .../importer/lsc2020/MetaImporter.java | 3 +- .../lsc2020/MyscealTagImportHandler.java | 3 +- .../importer/lsc2020/MyscealTagImporter.java | 25 +- .../importer/lsc2020/OCRImportHandler.java | 52 +- .../importer/lsc2020/OCRImporter.java | 79 +- .../lsc2020/ProcessingMetaImportHandler.java | 4 +- .../lsc2020/ProcessingMetaImporter.java | 574 ++++---- .../lsc2020/SpatialImportHandler.java | 5 +- .../importer/lsc2020/SpatialImporter.java | 38 +- .../lsc2020/TemporalImportHandler.java | 9 +- .../importer/lsc2020/TemporalImporter.java | 22 +- .../VisualConceptTagImportHandler.java | 35 +- .../lsc2020/VisualConceptTagImporter.java | 325 ++--- .../vbs2019/AudioTranscriptImportHandler.java | 7 +- .../vbs2019/AudioTranscriptionImporter.java | 13 +- .../vbs2019/CaptionTextImportHandler.java | 7 +- .../importer/vbs2019/CaptionTextImporter.java | 15 +- .../vbs2019/GoogleVisionImporter.java | 416 +++--- .../importer/vbs2019/MLTFeatureImporter.java | 9 +- .../vbs2019/MLTFeaturesImportHandler.java | 3 +- .../vbs2019/ObjectMetadataImportHandler.java | 1 - .../vbs2019/ObjectMetadataImporter.java | 188 +-- .../vbs2019/gvision/GoogleVisionCategory.java | 2 +- .../v3c1analysis/ColorlabelImporter.java | 4 +- .../listener/RetrievalResultCSVExporter.java | 32 +- .../listener/RetrievalResultListener.java | 8 +- .../monitoring/ImportTaskMonitor.java | 1 - .../PrometheusExtractionTaskMonitor.java | 3 +- .../monitoring/PrometheusServer.java | 1 - .../run/ExtractionCompleteListener.java | 25 +- .../run/ExtractionContainerProvider.java | 13 +- .../standalone/run/ExtractionDispatcher.java | 5 +- .../run/ExtractionItemContainer.java | 9 +- .../run/ExtractionItemProcessor.java | 1 - .../run/GenericExtractionItemHandler.java | 1 - .../ExtractionContainerProviderFactory.java | 12 +- .../run/path/NoContainerProvider.java | 6 +- .../run/path/SessionContainerProvider.java | 25 +- .../run/path/SingletonContainerProvider.java | 8 +- .../TreeWalkContainerIteratorProvider.java | 23 +- .../runtime/ContinuousQueryDispatcher.java | 18 +- .../runtime/ExecutionTimeCounter.java | 3 +- .../runtime/ExtractionPipeline.java | 414 +++--- .../standalone/runtime/RetrievalTask.java | 4 +- .../vitrivr/cineast/standalone/util/CLI.java | 184 ++- .../util/ContinuousRetrievalLogic.java | 9 +- cineast-runtime/src/main/resources/log4j2.xml | 62 +- 611 files changed, 29620 insertions(+), 29149 deletions(-) diff --git a/build.gradle b/build.gradle index 615fb9c73..4bfecbd8a 100644 --- a/build.gradle +++ b/build.gradle @@ -117,7 +117,7 @@ task generateOpenApiSpecs(type: JavaExec) { args("${config}") } -task generateOAS(type: Download){ +task generateOAS(type: Download) { /* This requires a locally running Cineast */ src 'http://localhost:4567/openapi-specs' dest "${project.projectDir}/docs/openapi.json" diff --git a/cineast-api/build.gradle b/cineast-api/build.gradle index 14c6e540c..a1c6ddfa8 100644 --- a/cineast-api/build.gradle +++ b/cineast-api/build.gradle @@ -89,7 +89,7 @@ configurations.all { resolutionStrategy.cacheChangingModulesFor 0, 'seconds' } -jar{ +jar { manifest { attributes 'Main-Class': 'org.vitrivr.cineast.api.Main' attributes 'Multi-Release': 'true' @@ -103,7 +103,7 @@ shadowJar { dependencies { implementation project(':cineast-runtime') - implementation("io.javalin:javalin-bundle:$version_javalin"){ + implementation("io.javalin:javalin-bundle:$version_javalin") { exclude group: 'ch.qos.logback', module: 'logback-classic' } implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastExtractionService.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastExtractionService.java index ee0d21547..775bd5642 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastExtractionService.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastExtractionService.java @@ -3,5 +3,4 @@ public class CineastExtractionService extends CineastExtractGrpc.CineastExtractImplBase { - } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastManagementService.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastManagementService.java index bd42b5ea7..a491da7d9 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastManagementService.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastManagementService.java @@ -4,10 +4,10 @@ public class CineastManagementService extends CineastManageGrpc.CineastManageImplBase { - @Override - public void ping(CineastGrpc.TimeStamp timeStamp, StreamObserver responseObserver) { - responseObserver.onNext(CineastGrpc.TimeStamp.newBuilder().setTimestamp(System.currentTimeMillis()).build()); - responseObserver.onCompleted(); - } + @Override + public void ping(CineastGrpc.TimeStamp timeStamp, StreamObserver responseObserver) { + responseObserver.onNext(CineastGrpc.TimeStamp.newBuilder().setTimestamp(System.currentTimeMillis()).build()); + responseObserver.onCompleted(); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastQueryService.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastQueryService.java index a5ddd4b7c..f27a6a3ee 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastQueryService.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastQueryService.java @@ -3,7 +3,12 @@ import io.grpc.stub.StreamObserver; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -29,344 +34,338 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.util.ContinuousRetrievalLogic; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - public class CineastQueryService extends CineastQueryGrpc.CineastQueryImplBase { - private static final int DEFAULT_NEIGHBORING_SEGMENTS = 10; - - private final ContinuousRetrievalLogic continuousRetrievalLogic; + private static final int DEFAULT_NEIGHBORING_SEGMENTS = 10; - private static final Logger LOGGER = LogManager.getLogger(); + private final ContinuousRetrievalLogic continuousRetrievalLogic; - public CineastQueryService(ContinuousRetrievalLogic continuousRetrievalLogic) { - this.continuousRetrievalLogic = continuousRetrievalLogic; - } + private static final Logger LOGGER = LogManager.getLogger(); - @Override - public void getMediaObjects(CineastGrpc.MediaObjectIdList request, StreamObserver responseObserver) { + public CineastQueryService(ContinuousRetrievalLogic continuousRetrievalLogic) { + this.continuousRetrievalLogic = continuousRetrievalLogic; + } - Set ids = request.getIdsList().stream().map(CineastGrpc.MediaObjectId::getId).collect(Collectors.toSet()); - MediaObjectReader reader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - Map objects = reader.lookUpObjects(ids); + @Override + public void getMediaObjects(CineastGrpc.MediaObjectIdList request, StreamObserver responseObserver) { - CineastGrpc.MediaObjectQueryResult result = CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects( - objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList()) - ).build(); + Set ids = request.getIdsList().stream().map(CineastGrpc.MediaObjectId::getId).collect(Collectors.toSet()); + MediaObjectReader reader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + Map objects = reader.lookUpObjects(ids); - responseObserver.onNext(result); - responseObserver.onCompleted(); - } + CineastGrpc.MediaObjectQueryResult result = CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects( + objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList()) + ).build(); - @Override - public void getMediaSegments(CineastGrpc.MediaSegmentIdList request, StreamObserver responseObserver) { + responseObserver.onNext(result); + responseObserver.onCompleted(); + } - Set ids = request.getIdsList().stream().map(CineastGrpc.MediaSegmentId::getId).collect(Collectors.toSet()); - MediaSegmentReader reader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - Map segments = reader.lookUpSegments(ids); + @Override + public void getMediaSegments(CineastGrpc.MediaSegmentIdList request, StreamObserver responseObserver) { - CineastGrpc.MediaSegmentQueryResult result = CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( - segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) - ).build(); + Set ids = request.getIdsList().stream().map(CineastGrpc.MediaSegmentId::getId).collect(Collectors.toSet()); + MediaSegmentReader reader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + Map segments = reader.lookUpSegments(ids); - responseObserver.onNext(result); - responseObserver.onCompleted(); - } + CineastGrpc.MediaSegmentQueryResult result = CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( + segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) + ).build(); - @Override - public void getMediaSegmentScores(CineastGrpc.Query query, StreamObserver responseObserver) { + responseObserver.onNext(result); + responseObserver.onCompleted(); + } - List stages = QueryContainerUtil.query(query); + @Override + public void getMediaSegmentScores(CineastGrpc.Query query, StreamObserver responseObserver) { - HashSet relevantSegments = new HashSet<>(); + List stages = QueryContainerUtil.query(query); - stages: - for (int i = 0; i < stages.size(); ++i) { + HashSet relevantSegments = new HashSet<>(); - QueryStage stage = stages.get(i); - boolean lastStage = i == stages.size() - 1; + stages: + for (int i = 0; i < stages.size(); ++i) { - List terms = stage.getQueryTerms(); - QueryConfig stageConfig = QueryConfig.clone(stage.getQueryConfig()); - stageConfig.addRelevantSegmentIds(relevantSegments); - relevantSegments.clear(); + QueryStage stage = stages.get(i); + boolean lastStage = i == stages.size() - 1; - for (QueryTerm term : terms) { + List terms = stage.getQueryTerms(); + QueryConfig stageConfig = QueryConfig.clone(stage.getQueryConfig()); + stageConfig.addRelevantSegmentIds(relevantSegments); + relevantSegments.clear(); - for (String category : term.getCategories()) { + for (QueryTerm term : terms) { - ReadableQueryConfig queryConfig = stageConfig.withChangesFrom(term.getQueryConfig()); + for (String category : term.getCategories()) { - List results = QueryUtil.retrieve(continuousRetrievalLogic, term.getContainer(), queryConfig, category); + ReadableQueryConfig queryConfig = stageConfig.withChangesFrom(term.getQueryConfig()); - if (lastStage) { - responseObserver.onNext(QueryContainerUtil.similarityQueryResult( - term.getQueryConfig().getQueryId().toString(), - category, - results - )); - } else { + List results = QueryUtil.retrieve(continuousRetrievalLogic, term.getContainer(), queryConfig, category); - if (results.isEmpty()) { //no more results left - break stages; - } + if (lastStage) { + responseObserver.onNext(QueryContainerUtil.similarityQueryResult( + term.getQueryConfig().getQueryId().toString(), + category, + results + )); + } else { - results.stream().forEach(x -> relevantSegments.add(x.key)); - } - } + if (results.isEmpty()) { //no more results left + break stages; } - } - responseObserver.onCompleted(); + results.stream().forEach(x -> relevantSegments.add(x.key)); + } + } + } } - //TODO This has enormous code duplication with the TemporalQueryMessageHandler - - @Override - public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver responseObserver) { - StopWatch watch = StopWatch.createStarted(); - - MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - - Set sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>(); - - CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig(); - ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config); - QueryConfig qconf = new QueryConfig(rqconf); - - /* Prepare QueryConfig (so as to obtain a QueryId). */ - final String uuid = qconf.getQueryId().toString(); - final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults()); - qconf.setMaxResults(max); - final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule(); - qconf.setResultsPerModule(resultsPerModule); - - List metadataRetrievalThreads = new ArrayList<>(); - - /* We iterate over all components independently, because they have a temporal context.*/ - for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) { - List stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx)); - - /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/ - QueryConfig stageQConf = QueryConfig.clone(qconf); - - /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */ - HashSet relevantSegments = new HashSet<>(); - - /* Store for each queryterm per category all results to be sent at a later time */ - List>> cache = new ArrayList<>(); - - /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */ - for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) { - /* Initalize stage with this hashmap */ - cache.add(stageIndex, new HashMap<>()); - - QueryStage stage = stages.get(stageIndex); - - List qtThreads = new ArrayList<>(); - - /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage. - * The list is only updated once we've iterated over all terms - */ - for (int i = 0; i < stage.getQueryTerms().size(); i++) { - QueryTerm qt = stage.getQueryTerms().get(i); - - final int finalContainerIdx = containerIdx; - final int finalStageIndex = stageIndex; - Thread qtRetrievalThread = new Thread(() -> { - - /* Prepare QueryTerm and perform sanity-checks */ - if (qt == null) { - /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */ - LOGGER.warn("QueryTerm was null for stage {}", stage); - return; - } - AbstractQueryTermContainer qc = qt.getContainer(); - if (qc == null) { - LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it"); - return; - } - - List categoryThreads = new ArrayList<>(); - - /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */ - for (String category : qt.getCategories()) { - /* Merge partial results with score-map */ - List scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf); - - /* Transform raw results into list of StringDoublePairs (segmentId -> score) */ - final List results = scores.stream() - .map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())) - .filter(p -> p.value > 0d) - .sorted(StringDoublePair.COMPARATOR) - .limit(max) - .collect(Collectors.toList()); - - if (results.isEmpty()) { - LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage); - } - if (cache.get(finalStageIndex).containsKey(category)) { - LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex); - } - cache.get(finalStageIndex).put(category, results); - results.forEach(res -> relevantSegments.add(res.key)); - LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS)); - - /* If this is the last stage, we can send relevant results per category back to the UI. - * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage - */ - if (finalStageIndex == stages.size() - 1) { - /* Finalize and submit per-container results */ - responseObserver.onNext( - QueryContainerUtil.queryResult( - QueryContainerUtil.similarityQueryResult( - qt.getQueryConfig().getQueryId().toString(), - category, - results - ))); - - List segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList()); - if (segmentIds.isEmpty()) { - continue; - } - - Map segments = mediaSegmentReader.lookUpSegments(segmentIds); - - responseObserver.onNext( - QueryContainerUtil.queryResult( - CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( - segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) - ).build() - ) - ); - - List segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds); - responseObserver.onNext( - QueryContainerUtil.queryResult( - CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData( - segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList()) - ).build() - ) - ); - - sentSegmentIds.addAll(segmentIds); - - List objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList()); - if (objectIds.isEmpty()) { - continue; - } - Map objects = mediaObjectReader.lookUpObjects(objectIds); - - responseObserver.onNext( - QueryContainerUtil.queryResult( - CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects( - objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList()) - ).build() - ) - ); - - List objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds); - responseObserver.onNext( - QueryContainerUtil.queryResult( - CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData( - objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList()) - ).build() - ) - ); - - sentObjectIds.addAll(objectIds); - - } - } - /* We're done for this querycontainer */ - }); - qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories()); //TODO Better name - qtThreads.add(qtRetrievalThread); - qtRetrievalThread.start(); - } + responseObserver.onCompleted(); + } - for (Thread thread : qtThreads) { - try { - thread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } + //TODO This has enormous code duplication with the TemporalQueryMessageHandler - /* After we are done with a stage, we add all relevant segments to the config for the next stage. */ - if (relevantSegments.size() == 0) { - LOGGER.warn("No relevant segments anymore, aborting staged querying"); - /* Clear relevant segments (there are none) */ - stageQConf.setRelevantSegmentIds(relevantSegments); - break; - } - stageQConf.setRelevantSegmentIds(relevantSegments); - relevantSegments.clear(); + @Override + public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver responseObserver) { + StopWatch watch = StopWatch.createStarted(); + + MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + + Set sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>(); + + CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig(); + ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config); + QueryConfig qconf = new QueryConfig(rqconf); + + /* Prepare QueryConfig (so as to obtain a QueryId). */ + final String uuid = qconf.getQueryId().toString(); + final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults()); + qconf.setMaxResults(max); + final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule(); + qconf.setResultsPerModule(resultsPerModule); + + List metadataRetrievalThreads = new ArrayList<>(); + + /* We iterate over all components independently, because they have a temporal context.*/ + for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) { + List stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx)); + + /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/ + QueryConfig stageQConf = QueryConfig.clone(qconf); + + /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */ + HashSet relevantSegments = new HashSet<>(); + + /* Store for each queryterm per category all results to be sent at a later time */ + List>> cache = new ArrayList<>(); + + /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */ + for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) { + /* Initalize stage with this hashmap */ + cache.add(stageIndex, new HashMap<>()); + + QueryStage stage = stages.get(stageIndex); + + List qtThreads = new ArrayList<>(); + + /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage. + * The list is only updated once we've iterated over all terms + */ + for (int i = 0; i < stage.getQueryTerms().size(); i++) { + QueryTerm qt = stage.getQueryTerms().get(i); + + final int finalContainerIdx = containerIdx; + final int finalStageIndex = stageIndex; + Thread qtRetrievalThread = new Thread(() -> { + + /* Prepare QueryTerm and perform sanity-checks */ + if (qt == null) { + /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */ + LOGGER.warn("QueryTerm was null for stage {}", stage); + return; + } + AbstractQueryTermContainer qc = qt.getContainer(); + if (qc == null) { + LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it"); + return; } - /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */ - for (int stageIndex = 0; stageIndex < stages.size()-1; stageIndex++) { - cache.get(stageIndex).forEach((category, results) -> { - results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key)); - - responseObserver.onNext( - QueryContainerUtil.queryResult( - QueryContainerUtil.similarityQueryResult( - uuid, //TODO This assumes that all queries in a temporalquery have the same uuid - category, - results - ))); - }); + List categoryThreads = new ArrayList<>(); + + /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */ + for (String category : qt.getCategories()) { + /* Merge partial results with score-map */ + List scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf); + + /* Transform raw results into list of StringDoublePairs (segmentId -> score) */ + final List results = scores.stream() + .map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())) + .filter(p -> p.value > 0d) + .sorted(StringDoublePair.COMPARATOR) + .limit(max) + .collect(Collectors.toList()); + + if (results.isEmpty()) { + LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage); + } + if (cache.get(finalStageIndex).containsKey(category)) { + LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex); + } + cache.get(finalStageIndex).put(category, results); + results.forEach(res -> relevantSegments.add(res.key)); + LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS)); + + /* If this is the last stage, we can send relevant results per category back to the UI. + * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage + */ + if (finalStageIndex == stages.size() - 1) { + /* Finalize and submit per-container results */ + responseObserver.onNext( + QueryContainerUtil.queryResult( + QueryContainerUtil.similarityQueryResult( + qt.getQueryConfig().getQueryId().toString(), + category, + results + ))); + + List segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList()); + if (segmentIds.isEmpty()) { + continue; + } + + Map segments = mediaSegmentReader.lookUpSegments(segmentIds); + + responseObserver.onNext( + QueryContainerUtil.queryResult( + CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( + segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) + ).build() + ) + ); + + List segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds); + responseObserver.onNext( + QueryContainerUtil.queryResult( + CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData( + segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList()) + ).build() + ) + ); + + sentSegmentIds.addAll(segmentIds); + + List objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList()); + if (objectIds.isEmpty()) { + continue; + } + Map objects = mediaObjectReader.lookUpObjects(objectIds); + + responseObserver.onNext( + QueryContainerUtil.queryResult( + CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects( + objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList()) + ).build() + ) + ); + + List objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds); + responseObserver.onNext( + QueryContainerUtil.queryResult( + CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData( + objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList()) + ).build() + ) + ); + + sentObjectIds.addAll(objectIds); + + } } + /* We're done for this querycontainer */ + }); + qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories()); //TODO Better name + qtThreads.add(qtRetrievalThread); + qtRetrievalThread.start(); + } - /* There should be no carry-over from this block since temporal queries are executed independently */ + for (Thread thread : qtThreads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } - /* At this point, all StagedQueries have been executed for this TemporalQuery. - * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do. - */ + /* After we are done with a stage, we add all relevant segments to the config for the next stage. */ + if (relevantSegments.size() == 0) { + LOGGER.warn("No relevant segments anymore, aborting staged querying"); + /* Clear relevant segments (there are none) */ + stageQConf.setRelevantSegmentIds(relevantSegments); + break; + } + stageQConf.setRelevantSegmentIds(relevantSegments); + relevantSegments.clear(); + } + + /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */ + for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) { + cache.get(stageIndex).forEach((category, results) -> { + results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key)); + + responseObserver.onNext( + QueryContainerUtil.queryResult( + QueryContainerUtil.similarityQueryResult( + uuid, //TODO This assumes that all queries in a temporalquery have the same uuid + category, + results + ))); + }); + } + + /* There should be no carry-over from this block since temporal queries are executed independently */ + } - responseObserver.onCompleted(); + /* At this point, all StagedQueries have been executed for this TemporalQuery. + * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do. + */ - mediaSegmentReader.close(); - mediaObjectReader.close(); - segmentMetadataReader.close(); - watch.stop(); - LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS)); + responseObserver.onCompleted(); - } + mediaSegmentReader.close(); + mediaObjectReader.close(); + segmentMetadataReader.close(); + watch.stop(); + LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS)); - @Override - public void getNeighboringSegments(CineastGrpc.MediaSegmentIdList request, StreamObserver responseObserver) { + } - MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - Set ids = request.getIdsList().stream().map(CineastGrpc.MediaSegmentId::getId).collect(Collectors.toSet()); - Map descriptors = mediaSegmentReader.lookUpSegments(ids); + @Override + public void getNeighboringSegments(CineastGrpc.MediaSegmentIdList request, StreamObserver responseObserver) { - int range = QueryContainerUtil.queryConfig(request.getQueryConfig()).getMaxResults().orElse(DEFAULT_NEIGHBORING_SEGMENTS) / 2; + MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); + Set ids = request.getIdsList().stream().map(CineastGrpc.MediaSegmentId::getId).collect(Collectors.toSet()); + Map descriptors = mediaSegmentReader.lookUpSegments(ids); - if (range > 0){ - Set results = new HashSet<>( 2 * range * descriptors.size()); + int range = QueryContainerUtil.queryConfig(request.getQueryConfig()).getMaxResults().orElse(DEFAULT_NEIGHBORING_SEGMENTS) / 2; - for (MediaSegmentDescriptor d : descriptors.values()) { - results.addAll(mediaSegmentReader.lookUpSegmentsByNumberRange(d.getObjectId(), d.getSequenceNumber() - range, d.getSequenceNumber() + range)); - } + if (range > 0) { + Set results = new HashSet<>(2 * range * descriptors.size()); - CineastGrpc.MediaSegmentQueryResult result = CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( - results.stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) - ).build(); - responseObserver.onNext(result); + for (MediaSegmentDescriptor d : descriptors.values()) { + results.addAll(mediaSegmentReader.lookUpSegmentsByNumberRange(d.getObjectId(), d.getSequenceNumber() - range, d.getSequenceNumber() + range)); + } - } + CineastGrpc.MediaSegmentQueryResult result = CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments( + results.stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList()) + ).build(); + responseObserver.onNext(result); - responseObserver.onCompleted(); - mediaSegmentReader.close(); } + + responseObserver.onCompleted(); + mediaSegmentReader.close(); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryStage.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryStage.java index 9e8fc0a44..9ca93bd04 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryStage.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryStage.java @@ -1,27 +1,26 @@ package org.vitrivr.cineast.api.grpc.data; -import org.vitrivr.cineast.core.config.ReadableQueryConfig; - import java.util.ArrayList; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; public class QueryStage { - private final ArrayList queryTerms = new ArrayList<>(); - private final ReadableQueryConfig queryConfig; + private final ArrayList queryTerms = new ArrayList<>(); + private final ReadableQueryConfig queryConfig; - public QueryStage(Iterable queryTerms, ReadableQueryConfig queryConfig) { - for (QueryTerm term : queryTerms) { - this.queryTerms.add(term); - } - this.queryConfig = queryConfig; + public QueryStage(Iterable queryTerms, ReadableQueryConfig queryConfig) { + for (QueryTerm term : queryTerms) { + this.queryTerms.add(term); } + this.queryConfig = queryConfig; + } - public ArrayList getQueryTerms() { - return queryTerms; - } + public ArrayList getQueryTerms() { + return queryTerms; + } - public ReadableQueryConfig getQueryConfig() { - return queryConfig; - } + public ReadableQueryConfig getQueryConfig() { + return queryConfig; + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryTerm.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryTerm.java index 892adaf91..e66b531b6 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryTerm.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryTerm.java @@ -1,40 +1,39 @@ package org.vitrivr.cineast.api.grpc.data; -import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; - import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; public class QueryTerm { - private final AbstractQueryTermContainer container; - private final ReadableQueryConfig queryConfig; - private final float weight; - private final List categories = new ArrayList<>(); - - public QueryTerm(AbstractQueryTermContainer container, ReadableQueryConfig queryConfig, float weight, Collection categories){ - this.container = container; - this.queryConfig = queryConfig; - this.weight = weight; - this.categories.addAll(categories); - } - - public AbstractQueryTermContainer getContainer() { - return container; - } - - public ReadableQueryConfig getQueryConfig() { - return queryConfig; - } - - public float getWeight() { - return weight; - } - - public List getCategories() { - return categories; - } + private final AbstractQueryTermContainer container; + private final ReadableQueryConfig queryConfig; + private final float weight; + private final List categories = new ArrayList<>(); + + public QueryTerm(AbstractQueryTermContainer container, ReadableQueryConfig queryConfig, float weight, Collection categories) { + this.container = container; + this.queryConfig = queryConfig; + this.weight = weight; + this.categories.addAll(categories); + } + + public AbstractQueryTermContainer getContainer() { + return container; + } + + public ReadableQueryConfig getQueryConfig() { + return queryConfig; + } + + public float getWeight() { + return weight; + } + + public List getCategories() { + return categories; + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaObjectUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaObjectUtil.java index 9db047554..afc3e42cd 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaObjectUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaObjectUtil.java @@ -6,28 +6,28 @@ public class MediaObjectUtil { - public static CineastGrpc.MediaObject fromMediaObjectDescriptor(MediaObjectDescriptor descriptor) { - if (descriptor == null) { - return null; - } + public static CineastGrpc.MediaObject fromMediaObjectDescriptor(MediaObjectDescriptor descriptor) { + if (descriptor == null) { + return null; + } - CineastGrpc.MediaObject.Builder builder = CineastGrpc.MediaObject.newBuilder(); + CineastGrpc.MediaObject.Builder builder = CineastGrpc.MediaObject.newBuilder(); - builder.setObjectId(mediaObjectId(descriptor.getObjectId())); - builder.setName(descriptor.getName()); - builder.setPath(descriptor.getPath()); - builder.setType(mediaType(descriptor.getMediatype())); - builder.setContentUrl(descriptor.getContentURL()); + builder.setObjectId(mediaObjectId(descriptor.getObjectId())); + builder.setName(descriptor.getName()); + builder.setPath(descriptor.getPath()); + builder.setType(mediaType(descriptor.getMediatype())); + builder.setContentUrl(descriptor.getContentURL()); - return builder.build(); - } + return builder.build(); + } - public static CineastGrpc.MediaObjectId mediaObjectId(String id){ - return CineastGrpc.MediaObjectId.newBuilder().setId(id).build(); - } + public static CineastGrpc.MediaObjectId mediaObjectId(String id) { + return CineastGrpc.MediaObjectId.newBuilder().setId(id).build(); + } - public static CineastGrpc.MediaObject.MediaType mediaType(MediaType type){ - return CineastGrpc.MediaObject.MediaType.forNumber(type.getId()); - } + public static CineastGrpc.MediaObject.MediaType mediaType(MediaType type) { + return CineastGrpc.MediaObject.MediaType.forNumber(type.getId()); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaSegmentUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaSegmentUtil.java index 5836337f0..fbd473c97 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaSegmentUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/MediaSegmentUtil.java @@ -5,24 +5,24 @@ public class MediaSegmentUtil { - public static CineastGrpc.MediaSegment fromMediaSegmentDescriptor(MediaSegmentDescriptor descriptor){ + public static CineastGrpc.MediaSegment fromMediaSegmentDescriptor(MediaSegmentDescriptor descriptor) { - CineastGrpc.MediaSegment.Builder builder = CineastGrpc.MediaSegment.newBuilder(); + CineastGrpc.MediaSegment.Builder builder = CineastGrpc.MediaSegment.newBuilder(); - builder.setObjectId(MediaObjectUtil.mediaObjectId(descriptor.getObjectId())); - builder.setSegmentId(mediaSegmentId(descriptor.getSegmentId())); - builder.setStart(descriptor.getStart()); - builder.setEnd(descriptor.getEnd()); - builder.setStartAbs(descriptor.getStartabs()); - builder.setEndAbs(descriptor.getEndabs()); - builder.setSequenceNumber(descriptor.getSequenceNumber()); + builder.setObjectId(MediaObjectUtil.mediaObjectId(descriptor.getObjectId())); + builder.setSegmentId(mediaSegmentId(descriptor.getSegmentId())); + builder.setStart(descriptor.getStart()); + builder.setEnd(descriptor.getEnd()); + builder.setStartAbs(descriptor.getStartabs()); + builder.setEndAbs(descriptor.getEndabs()); + builder.setSequenceNumber(descriptor.getSequenceNumber()); - return builder.build(); + return builder.build(); - } + } - public static CineastGrpc.MediaSegmentId mediaSegmentId(String id) { - return CineastGrpc.MediaSegmentId.newBuilder().setId(id).build(); - } + public static CineastGrpc.MediaSegmentId mediaSegmentId(String id) { + return CineastGrpc.MediaSegmentId.newBuilder().setId(id).build(); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/QueryContainerUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/QueryContainerUtil.java index 200e40c56..8c30e2ce7 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/QueryContainerUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/QueryContainerUtil.java @@ -1,6 +1,15 @@ package org.vitrivr.cineast.api.grpc.util; +import static org.vitrivr.cineast.core.config.ReadableQueryConfig.Hints; + import georegression.struct.point.Point2D_F32; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.api.grpc.CineastGrpc; @@ -15,256 +24,296 @@ import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; -import org.vitrivr.cineast.core.data.providers.primitive.*; -import org.vitrivr.cineast.core.data.query.containers.*; +import org.vitrivr.cineast.core.data.providers.primitive.BooleanTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.DoubleTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.FloatTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.IntTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.LongTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.NothingProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.AudioQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.BooleanQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.IdQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.ImageQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.InstantQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.LocationQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.MotionQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.SemanticMapQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.TagQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.TextQueryTermContainer; import org.vitrivr.cineast.core.db.BooleanExpression; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.util.LogHelper; -import javax.imageio.ImageIO; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.vitrivr.cineast.core.config.ReadableQueryConfig.Hints; - public class QueryContainerUtil { - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); - public static AudioQueryTermContainer audioQueryContainer(CineastGrpc.AudioQueryContainer container) { - return new AudioQueryTermContainer(container.getAudioFramesList().stream().map(QueryContainerUtil::audioFrame).collect(Collectors.toList())); - } + public static AudioQueryTermContainer audioQueryContainer(CineastGrpc.AudioQueryContainer container) { + return new AudioQueryTermContainer(container.getAudioFramesList().stream().map(QueryContainerUtil::audioFrame).collect(Collectors.toList())); + } - public static AudioFrame audioFrame(CineastGrpc.AudioFrame frame) { - AudioDescriptor descriptor = new AudioDescriptor(frame.getSamplingRate(), frame.getChannels(), frame.getDuration()); - return new AudioFrame(frame.getIdx(), frame.getTimestamp(), frame.getData().toByteArray(), descriptor); - } + public static AudioFrame audioFrame(CineastGrpc.AudioFrame frame) { + AudioDescriptor descriptor = new AudioDescriptor(frame.getSamplingRate(), frame.getChannels(), frame.getDuration()); + return new AudioFrame(frame.getIdx(), frame.getTimestamp(), frame.getData().toByteArray(), descriptor); + } - public static BooleanQueryTermContainer booleanQueryContainer(CineastGrpc.BooleanQueryContainer container) { + public static BooleanQueryTermContainer booleanQueryContainer(CineastGrpc.BooleanQueryContainer container) { - return new BooleanQueryTermContainer( + return new BooleanQueryTermContainer( container.getExpressionsList().stream().map(QueryContainerUtil::booleanExpression).collect(Collectors.toList()) - ); - - } - - public static BooleanExpression booleanExpression(CineastGrpc.BooleanExpression expression) { - return new BooleanExpression( - expression.getAttribute(), - relationalOperator(expression.getOperator()), - expression.getValuesList().stream().map(QueryContainerUtil::primitiveTypeProvider).collect(Collectors.toList()) - ); - } - - public static RelationalOperator relationalOperator(CineastGrpc.BooleanExpression.Operator operator) { - switch (operator){ - case EQ: return RelationalOperator.EQ; - case NEQ: return RelationalOperator.NEQ; - case GEQ: return RelationalOperator.GEQ; - case LEQ: return RelationalOperator.LEQ; - case GREATER: return RelationalOperator.GREATER; - case LESS: return RelationalOperator.LESS; - case BETWEEN: return RelationalOperator.BETWEEN; - case LIKE: return RelationalOperator.LIKE; - case NLIKE: return RelationalOperator.NLIKE; - case ISNULL: return RelationalOperator.ISNULL; - case ISNOTNULL: return RelationalOperator.ISNOTNULL; - case IN: return RelationalOperator.IN; - case UNRECOGNIZED: return null; - /* TODO: Support MATCH operator. */ - } - + ); + + } + + public static BooleanExpression booleanExpression(CineastGrpc.BooleanExpression expression) { + return new BooleanExpression( + expression.getAttribute(), + relationalOperator(expression.getOperator()), + expression.getValuesList().stream().map(QueryContainerUtil::primitiveTypeProvider).collect(Collectors.toList()) + ); + } + + public static RelationalOperator relationalOperator(CineastGrpc.BooleanExpression.Operator operator) { + switch (operator) { + case EQ: + return RelationalOperator.EQ; + case NEQ: + return RelationalOperator.NEQ; + case GEQ: + return RelationalOperator.GEQ; + case LEQ: + return RelationalOperator.LEQ; + case GREATER: + return RelationalOperator.GREATER; + case LESS: + return RelationalOperator.LESS; + case BETWEEN: + return RelationalOperator.BETWEEN; + case LIKE: + return RelationalOperator.LIKE; + case NLIKE: + return RelationalOperator.NLIKE; + case ISNULL: + return RelationalOperator.ISNULL; + case ISNOTNULL: + return RelationalOperator.ISNOTNULL; + case IN: + return RelationalOperator.IN; + case UNRECOGNIZED: return null; - } - - public static PrimitiveTypeProvider primitiveTypeProvider(CineastGrpc.PrimitiveType type) { - switch(type.getValueCase()) { - case STRINGVALUE: return new StringTypeProvider(type.getStringValue()); - case INTVALUE: return new IntTypeProvider(type.getIntValue()); - case LONGVALUE: return new LongTypeProvider(type.getLongValue()); - case FLOATVALUE: return new FloatTypeProvider(type.getFloatValue()); - case DOUBLEVALUE: return new DoubleTypeProvider(type.getDoubleValue()); - case BOOLEANVALUE: return new BooleanTypeProvider(type.getBooleanValue()); - case VALUE_NOT_SET: return new NothingProvider(); - } + /* TODO: Support MATCH operator. */ + } + + return null; + } + + public static PrimitiveTypeProvider primitiveTypeProvider(CineastGrpc.PrimitiveType type) { + switch (type.getValueCase()) { + case STRINGVALUE: + return new StringTypeProvider(type.getStringValue()); + case INTVALUE: + return new IntTypeProvider(type.getIntValue()); + case LONGVALUE: + return new LongTypeProvider(type.getLongValue()); + case FLOATVALUE: + return new FloatTypeProvider(type.getFloatValue()); + case DOUBLEVALUE: + return new DoubleTypeProvider(type.getDoubleValue()); + case BOOLEANVALUE: + return new BooleanTypeProvider(type.getBooleanValue()); + case VALUE_NOT_SET: return new NothingProvider(); } - - public static IdQueryTermContainer idQueryContainer(CineastGrpc.IdQueryContainer container) { - return new IdQueryTermContainer(container.getSegmentId().getId()); - } - - public static ImageQueryTermContainer imageQueryContainer(CineastGrpc.ImageQueryContainer container) { - try { - return new ImageQueryTermContainer( - ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray())) - ); - } catch (IOException e) { - LOGGER.error("Error in reading image from ImageQueryContainer: {}", LogHelper.getStackTrace(e)); - } - return null; - } - - public static InstantQueryTermContainer instantQueryContainer(CineastGrpc.InstantQueryContainer container) { - return new InstantQueryTermContainer(container.getInstant()); - } - - public static LocationQueryTermContainer locationQueryContainer(CineastGrpc.LocationQueryContainer container) { - return new LocationQueryTermContainer(Location.of(container.getLongitude(), container.getLatitude())); - } - - public static ModelQueryTermContainer modelQueryContainer(CineastGrpc.ModelQueryContainer container) { - //TODO figure out better mesh representation - return null; - } - - public static MotionQueryTermContainer motionQueryContainer(CineastGrpc.MotionQueryContainer container) { - MotionQueryTermContainer motionQueryContainer = new MotionQueryTermContainer(); - container.getBackgroundPathList().stream().forEach(path -> motionQueryContainer.addBgPath(motionPath(path))); - container.getForegroundPathList().stream().forEach(path -> motionQueryContainer.addPath(motionPath(path))); - return motionQueryContainer; - } - - public static LinkedList motionPath(CineastGrpc.MotionQueryContainer.MotionPath path) { - LinkedList list = new LinkedList<>(); - list.addAll(path.getPathList().stream().map(QueryContainerUtil::point).collect(Collectors.toList())); - return list; - } - - public static Point2D_F32 point(CineastGrpc.MotionQueryContainer.MotionPath.Point point) { - return new Point2D_F32(point.getX(), point.getY()); - } - - public static SemanticMapQueryTermContainer semanticMapQueryContainer(CineastGrpc.SemanticMapQueryContainer container) { - try { - return new SemanticMapQueryTermContainer(new SemanticMap( - ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray())), - container.getConceptsMap()) - ); - } catch (IOException e) { - LOGGER.error("Error in reading image from ImageQueryContainer: {}", LogHelper.getStackTrace(e)); - } - return null; - } - - public static TagQueryTermContainer tagQueryContainer(CineastGrpc.TagQueryContainer container) { - return null; //TODO do we even still need that one? - } - - public static TextQueryTermContainer textQueryContainer(CineastGrpc.TextQueryContainer container) { - return new TextQueryTermContainer(container.getText()); - } - - - public static AbstractQueryTermContainer queryTermContainer(CineastGrpc.QueryTerm term) { - switch(term.getContainerCase()){ - - case AUDIOQUERYCONTAINER: return audioQueryContainer(term.getAudioQueryContainer()); - case BOOLEANQUERYCONTAINER: return booleanQueryContainer(term.getBooleanQueryContainer()); - case IDQUERYCONTAINER: return idQueryContainer(term.getIdQueryContainer()); - case IMAGEQUERYCONTAINER: return imageQueryContainer(term.getImageQueryContainer()); - case INSTANTQUERYCONTAINER: return instantQueryContainer(term.getInstantQueryContainer()); - case LOCATIONQUERYCONTAINER: return locationQueryContainer(term.getLocationQueryContainer()); - case MODELQUERYCONTAINER: return modelQueryContainer(term.getModelQueryContainer()); - case MOTIONQUERYCONTAINER: return motionQueryContainer(term.getMotionQueryContainer()); - case SEMANTICMAPQUERYCONTAINER: return semanticMapQueryContainer(term.getSemanticMapQueryContainer()); - case TAGQUERYCONTAINER: return tagQueryContainer(term.getTagQueryContainer()); - case TEXTQUERYCONTAINER: return textQueryContainer(term.getTextQueryContainer()); - case CONTAINER_NOT_SET: return null; - } + return new NothingProvider(); + } + + public static IdQueryTermContainer idQueryContainer(CineastGrpc.IdQueryContainer container) { + return new IdQueryTermContainer(container.getSegmentId().getId()); + } + + public static ImageQueryTermContainer imageQueryContainer(CineastGrpc.ImageQueryContainer container) { + try { + return new ImageQueryTermContainer( + ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray())) + ); + } catch (IOException e) { + LOGGER.error("Error in reading image from ImageQueryContainer: {}", LogHelper.getStackTrace(e)); + } + return null; + } + + public static InstantQueryTermContainer instantQueryContainer(CineastGrpc.InstantQueryContainer container) { + return new InstantQueryTermContainer(container.getInstant()); + } + + public static LocationQueryTermContainer locationQueryContainer(CineastGrpc.LocationQueryContainer container) { + return new LocationQueryTermContainer(Location.of(container.getLongitude(), container.getLatitude())); + } + + public static ModelQueryTermContainer modelQueryContainer(CineastGrpc.ModelQueryContainer container) { + //TODO figure out better mesh representation + return null; + } + + public static MotionQueryTermContainer motionQueryContainer(CineastGrpc.MotionQueryContainer container) { + MotionQueryTermContainer motionQueryContainer = new MotionQueryTermContainer(); + container.getBackgroundPathList().stream().forEach(path -> motionQueryContainer.addBgPath(motionPath(path))); + container.getForegroundPathList().stream().forEach(path -> motionQueryContainer.addPath(motionPath(path))); + return motionQueryContainer; + } + + public static LinkedList motionPath(CineastGrpc.MotionQueryContainer.MotionPath path) { + LinkedList list = new LinkedList<>(); + list.addAll(path.getPathList().stream().map(QueryContainerUtil::point).collect(Collectors.toList())); + return list; + } + + public static Point2D_F32 point(CineastGrpc.MotionQueryContainer.MotionPath.Point point) { + return new Point2D_F32(point.getX(), point.getY()); + } + + public static SemanticMapQueryTermContainer semanticMapQueryContainer(CineastGrpc.SemanticMapQueryContainer container) { + try { + return new SemanticMapQueryTermContainer(new SemanticMap( + ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray())), + container.getConceptsMap()) + ); + } catch (IOException e) { + LOGGER.error("Error in reading image from ImageQueryContainer: {}", LogHelper.getStackTrace(e)); + } + return null; + } + + public static TagQueryTermContainer tagQueryContainer(CineastGrpc.TagQueryContainer container) { + return null; //TODO do we even still need that one? + } + + public static TextQueryTermContainer textQueryContainer(CineastGrpc.TextQueryContainer container) { + return new TextQueryTermContainer(container.getText()); + } + + + public static AbstractQueryTermContainer queryTermContainer(CineastGrpc.QueryTerm term) { + switch (term.getContainerCase()) { + + case AUDIOQUERYCONTAINER: + return audioQueryContainer(term.getAudioQueryContainer()); + case BOOLEANQUERYCONTAINER: + return booleanQueryContainer(term.getBooleanQueryContainer()); + case IDQUERYCONTAINER: + return idQueryContainer(term.getIdQueryContainer()); + case IMAGEQUERYCONTAINER: + return imageQueryContainer(term.getImageQueryContainer()); + case INSTANTQUERYCONTAINER: + return instantQueryContainer(term.getInstantQueryContainer()); + case LOCATIONQUERYCONTAINER: + return locationQueryContainer(term.getLocationQueryContainer()); + case MODELQUERYCONTAINER: + return modelQueryContainer(term.getModelQueryContainer()); + case MOTIONQUERYCONTAINER: + return motionQueryContainer(term.getMotionQueryContainer()); + case SEMANTICMAPQUERYCONTAINER: + return semanticMapQueryContainer(term.getSemanticMapQueryContainer()); + case TAGQUERYCONTAINER: + return tagQueryContainer(term.getTagQueryContainer()); + case TEXTQUERYCONTAINER: + return textQueryContainer(term.getTextQueryContainer()); + case CONTAINER_NOT_SET: return null; } - - public static ReadableQueryConfig queryConfig(CineastGrpc.QueryConfig queryConfig) { - List hints = new ArrayList(queryConfig.getHintsCount()); - for(String hint : queryConfig.getHintsList()){ - if(hint == null) { - continue; - } - - Hints h = null; - - try { - h = Hints.valueOf(hint); - }catch (IllegalArgumentException e){ - //ignore - } - if (h != null){ - hints.add(h); - } - } - - QueryConfig config = new QueryConfig(queryConfig.getQueryId().getId(), hints); - config.setMaxResults(queryConfig.getMaxResults()); - return config; - } - - public static QueryTerm queryTerm(CineastGrpc.QueryTerm term) { - return new QueryTerm( - queryTermContainer(term), - queryConfig(term.getConfig()), - term.getWeight(), - term.getCategoryList() - ); - } - - public static QueryStage queryStage(CineastGrpc.QueryStage queryStage) { - return new QueryStage( - queryStage.getTermsList().stream().map(QueryContainerUtil::queryTerm).collect(Collectors.toList()), - queryConfig(queryStage.getConfig()) - ); - } - - public static List query(CineastGrpc.Query query) { - return query.getStagesList().stream().map(QueryContainerUtil::queryStage).collect(Collectors.toList()); - } - - public static CineastGrpc.MediaSegmentId mediaSegmentId(String id) { - return CineastGrpc.MediaSegmentId.newBuilder().setId(id).build(); - } - - public static CineastGrpc.MediaSegmentIdScore mediaSegmentIdScore(StringDoublePair pair) { - return CineastGrpc.MediaSegmentIdScore.newBuilder().setId(mediaSegmentId(pair.key)).setScore(pair.value).build(); - } - - public static CineastGrpc.QueryId queryId(String id) { - return CineastGrpc.QueryId.newBuilder().setId(id).build(); - } - - public static CineastGrpc.SimilarityQueryResult similarityQueryResult(String queryId, String category, List pairs) { - return CineastGrpc.SimilarityQueryResult.newBuilder().setQueryId(queryId(queryId)).setCategory(category).addAllResults(pairs.stream().map(QueryContainerUtil::mediaSegmentIdScore).collect(Collectors.toList())).build(); - } - - public static CineastGrpc.QueryResult queryResult(CineastGrpc.SimilarityQueryResult result) { - return CineastGrpc.QueryResult.newBuilder().setSimilarityQueryResult(result).build(); - } - - public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaSegmentQueryResult result) { - return CineastGrpc.QueryResult.newBuilder().setMediaSegmentQueryResult(result).build(); - } - - public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaObjectQueryResult result) { - return CineastGrpc.QueryResult.newBuilder().setMediaObjectQueryResult(result).build(); - } - - public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult result) { - return CineastGrpc.QueryResult.newBuilder().setMediaSegmentMetaDataQueryResult(result).build(); - } - - public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaObjectMetaDataQueryResult result) { - return CineastGrpc.QueryResult.newBuilder().setMediaObjectMetaDataQueryResult(result).build(); - } - - public static CineastGrpc.MediaObjectMetaData mediaObjectMetaData(MediaObjectMetadataDescriptor descriptor){ - return CineastGrpc.MediaObjectMetaData.newBuilder().setDomain(descriptor.getDomain()).setKey(descriptor.getKey()).setValue(descriptor.getValue()).build(); - } - - public static CineastGrpc.MediaSegmentMetaData mediaSegmentMetaData(MediaSegmentMetadataDescriptor descriptor){ - return CineastGrpc.MediaSegmentMetaData.newBuilder().setDomain(descriptor.getDomain()).setKey(descriptor.getKey()).setValue(descriptor.getValue()).build(); - } + return null; + } + + public static ReadableQueryConfig queryConfig(CineastGrpc.QueryConfig queryConfig) { + List hints = new ArrayList(queryConfig.getHintsCount()); + for (String hint : queryConfig.getHintsList()) { + if (hint == null) { + continue; + } + + Hints h = null; + + try { + h = Hints.valueOf(hint); + } catch (IllegalArgumentException e) { + //ignore + } + if (h != null) { + hints.add(h); + } + } + + QueryConfig config = new QueryConfig(queryConfig.getQueryId().getId(), hints); + config.setMaxResults(queryConfig.getMaxResults()); + return config; + } + + public static QueryTerm queryTerm(CineastGrpc.QueryTerm term) { + return new QueryTerm( + queryTermContainer(term), + queryConfig(term.getConfig()), + term.getWeight(), + term.getCategoryList() + ); + } + + public static QueryStage queryStage(CineastGrpc.QueryStage queryStage) { + return new QueryStage( + queryStage.getTermsList().stream().map(QueryContainerUtil::queryTerm).collect(Collectors.toList()), + queryConfig(queryStage.getConfig()) + ); + } + + public static List query(CineastGrpc.Query query) { + return query.getStagesList().stream().map(QueryContainerUtil::queryStage).collect(Collectors.toList()); + } + + public static CineastGrpc.MediaSegmentId mediaSegmentId(String id) { + return CineastGrpc.MediaSegmentId.newBuilder().setId(id).build(); + } + + public static CineastGrpc.MediaSegmentIdScore mediaSegmentIdScore(StringDoublePair pair) { + return CineastGrpc.MediaSegmentIdScore.newBuilder().setId(mediaSegmentId(pair.key)).setScore(pair.value).build(); + } + + public static CineastGrpc.QueryId queryId(String id) { + return CineastGrpc.QueryId.newBuilder().setId(id).build(); + } + + public static CineastGrpc.SimilarityQueryResult similarityQueryResult(String queryId, String category, List pairs) { + return CineastGrpc.SimilarityQueryResult.newBuilder().setQueryId(queryId(queryId)).setCategory(category).addAllResults(pairs.stream().map(QueryContainerUtil::mediaSegmentIdScore).collect(Collectors.toList())).build(); + } + + public static CineastGrpc.QueryResult queryResult(CineastGrpc.SimilarityQueryResult result) { + return CineastGrpc.QueryResult.newBuilder().setSimilarityQueryResult(result).build(); + } + + public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaSegmentQueryResult result) { + return CineastGrpc.QueryResult.newBuilder().setMediaSegmentQueryResult(result).build(); + } + + public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaObjectQueryResult result) { + return CineastGrpc.QueryResult.newBuilder().setMediaObjectQueryResult(result).build(); + } + + public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult result) { + return CineastGrpc.QueryResult.newBuilder().setMediaSegmentMetaDataQueryResult(result).build(); + } + + public static CineastGrpc.QueryResult queryResult(CineastGrpc.MediaObjectMetaDataQueryResult result) { + return CineastGrpc.QueryResult.newBuilder().setMediaObjectMetaDataQueryResult(result).build(); + } + + public static CineastGrpc.MediaObjectMetaData mediaObjectMetaData(MediaObjectMetadataDescriptor descriptor) { + return CineastGrpc.MediaObjectMetaData.newBuilder().setDomain(descriptor.getDomain()).setKey(descriptor.getKey()).setValue(descriptor.getValue()).build(); + } + + public static CineastGrpc.MediaSegmentMetaData mediaSegmentMetaData(MediaSegmentMetadataDescriptor descriptor) { + return CineastGrpc.MediaSegmentMetaData.newBuilder().setDomain(descriptor.getDomain()).setKey(descriptor.getKey()).setValue(descriptor.getValue()).build(); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/NeighboringSegmentQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/NeighboringSegmentQuery.java index 25dbe7452..4f11c78b2 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/NeighboringSegmentQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/NeighboringSegmentQuery.java @@ -2,67 +2,66 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import org.vitrivr.cineast.api.messages.interfaces.MessageType; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; -import java.util.List; - /** * A {@link NeighboringSegmentQuery} represents a query for neighbors of a given segment ID. */ public class NeighboringSegmentQuery extends Query { - /** - * ID of the {@link MediaSegmentDescriptor} for which neighbors should be retrieved. - */ - private final String segmentId; + /** + * ID of the {@link MediaSegmentDescriptor} for which neighbors should be retrieved. + */ + private final String segmentId; - /** - * Number of neighbors that should be retrieved. - */ - private final int count; + /** + * Number of neighbors that should be retrieved. + */ + private final int count; - private final List metadataAccessSpec; + private final List metadataAccessSpec; - /** - * Constructor for the NeighboringSegmentQuery object. - * - * @param segmentId ID of the {@link MediaSegmentDescriptor}. - * @param count Number of neighbors to be retrieved. - * @param config The {@link QueryConfig}. May be null! - */ - @JsonCreator - public NeighboringSegmentQuery(@JsonProperty(value = "segmentId", required = true) String segmentId, - @JsonProperty(value = "count", required = false) Integer count, - @JsonProperty(value = "config", required = false) QueryConfig config, - @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec - ) { - super(config); - this.segmentId = segmentId; - this.count = count == null ? 3 : count; - this.metadataAccessSpec = metadataAccessSpec; - } + /** + * Constructor for the NeighboringSegmentQuery object. + * + * @param segmentId ID of the {@link MediaSegmentDescriptor}. + * @param count Number of neighbors to be retrieved. + * @param config The {@link QueryConfig}. May be null! + */ + @JsonCreator + public NeighboringSegmentQuery(@JsonProperty(value = "segmentId", required = true) String segmentId, + @JsonProperty(value = "count", required = false) Integer count, + @JsonProperty(value = "config", required = false) QueryConfig config, + @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec + ) { + super(config); + this.segmentId = segmentId; + this.count = count == null ? 3 : count; + this.metadataAccessSpec = metadataAccessSpec; + } - public List getMetadataAccessSpec() { - return metadataAccessSpec; - } + public List getMetadataAccessSpec() { + return metadataAccessSpec; + } - public String getSegmentId() { - return segmentId; - } + public String getSegmentId() { + return segmentId; + } - public int getCount() { - return count; - } + public int getCount() { + return count; + } - /** - * {@inheritDoc} - */ - @Override - public MessageType getMessageType() { - return MessageType.Q_NESEG; - } + /** + * {@inheritDoc} + */ + @Override + public MessageType getMessageType() { + return MessageType.Q_NESEG; + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryComponent.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryComponent.java index 53ddcd2cf..88e2b612c 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryComponent.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryComponent.java @@ -2,9 +2,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java index d6ad75e17..f9e891fc6 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java @@ -5,6 +5,7 @@ import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.AudioQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.BooleanQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.IdQueryTermContainer; @@ -13,7 +14,6 @@ import org.vitrivr.cineast.core.data.query.containers.LocationQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.MotionQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.ParameterisedLocationQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.SemanticMapQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.TagQueryTermContainer; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/SegmentQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/SegmentQuery.java index f557da68c..3d1c7ed38 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/SegmentQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/SegmentQuery.java @@ -2,55 +2,54 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import org.vitrivr.cineast.api.messages.interfaces.MessageType; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; -import java.util.List; - /** * A {@link SegmentQuery} represents a segment-query message, i.e. a lookup for a segment ID specified. */ public class SegmentQuery extends Query { - /** - * ID of the {@link MediaSegmentDescriptor} which should be retrieved. - */ - private final String segmentId; - - private final List metadataAccessSpec; - - - /** - * Constructor for {@link SegmentQuery} message. - * - * @param config The {@link QueryConfig} - */ - @JsonCreator - public SegmentQuery(@JsonProperty(value = "segmentId", required = true) String segmentId, - @JsonProperty(value = "config", required = false) QueryConfig config, - @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec - - ) { - super(config); - this.segmentId = segmentId; - this.metadataAccessSpec = metadataAccessSpec; - } - - public List getMetadataAccessSpec() { - return metadataAccessSpec; - } - - public String getSegmentId() { - return segmentId; - } - - /** - * {@inheritDoc} - */ - @Override - public MessageType getMessageType() { - return MessageType.Q_SEG; - } + /** + * ID of the {@link MediaSegmentDescriptor} which should be retrieved. + */ + private final String segmentId; + + private final List metadataAccessSpec; + + + /** + * Constructor for {@link SegmentQuery} message. + * + * @param config The {@link QueryConfig} + */ + @JsonCreator + public SegmentQuery(@JsonProperty(value = "segmentId", required = true) String segmentId, + @JsonProperty(value = "config", required = false) QueryConfig config, + @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec + + ) { + super(config); + this.segmentId = segmentId; + this.metadataAccessSpec = metadataAccessSpec; + } + + public List getMetadataAccessSpec() { + return metadataAccessSpec; + } + + public String getSegmentId() { + return segmentId; + } + + /** + * {@inheritDoc} + */ + @Override + public MessageType getMessageType() { + return MessageType.Q_SEG; + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindTagsForElementGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindTagsForElementGetHandler.java index 1dc3b8b36..8c831cb70 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindTagsForElementGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindTagsForElementGetHandler.java @@ -15,7 +15,7 @@ public class FindTagsForElementGetHandler implements GetRestHandler { - public static final String ROUTE = "find/feature/tags/by/id/{" + ID_QUALIFIER+"}"; + public static final String ROUTE = "find/feature/tags/by/id/{" + ID_QUALIFIER + "}"; @Override public TagIDsForElementQueryResult doGet(Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java index c3156dda0..d04340d23 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java @@ -18,7 +18,7 @@ public class FindObjectGetHandler implements GetRestHandler { - public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME +"}"; + public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME + "}"; @Override public MediaObjectMetadataQueryResult performPost(IdList ids, Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java index 44cbecd08..cdba7633f 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java @@ -15,7 +15,7 @@ public class FindObjectMetadataByKeyPostHandler implements ParsingPostRestHandler { - public static final String ROUTE = "find/metadata/with/{" + KEY_NAME+"}"; + public static final String ROUTE = "find/metadata/with/{" + KEY_NAME + "}"; @Override public MediaObjectMetadataQueryResult performPost(IdList ids, Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java index be3909b54..9fc17ac39 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java @@ -23,7 +23,7 @@ public class FindObjectMetadataFullyQualifiedGetHandler implements public static final String DOMAIN_NAME = "domain"; public static final String KEY_NAME = "key"; - public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + DOMAIN_NAME + "}/with/{" + KEY_NAME +"}"; + public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + DOMAIN_NAME + "}/with/{" + KEY_NAME + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataGetHandler.java index b0670e315..dc2c909ce 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataGetHandler.java @@ -13,7 +13,7 @@ public class FindObjectMetadataGetHandler implements GetRestHandler { - public static final String ROUTE = "find/metadata/by/id/{" + OBJECT_ID_NAME +"}"; + public static final String ROUTE = "find/metadata/by/id/{" + OBJECT_ID_NAME + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindSegmentMetadataGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindSegmentMetadataGetHandler.java index d36c35f99..adae60dd6 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindSegmentMetadataGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindSegmentMetadataGetHandler.java @@ -14,7 +14,7 @@ public class FindSegmentMetadataGetHandler implements GetRestHandler { - public static final String ROUTE = "find/metadata/by/segmentid/{" + ID_QUALIFIER +"}"; + public static final String ROUTE = "find/metadata/by/segmentid/{" + ID_QUALIFIER + "}"; @Override public MediaSegmentMetadataQueryResult doGet(Context ctx) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java index 7ef165908..d7930c7ba 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java @@ -16,7 +16,7 @@ public class FindSegmentsByIdGetHandler implements GetRestHandler { public static final String ID_NAME = "id"; - public static final String ROUTE = "find/segments/all/object/{" + ID_NAME +"}"; + public static final String ROUTE = "find/segments/all/object/{" + ID_NAME + "}"; @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/EndSessionHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/EndSessionHandler.java index 0e5b6231a..196594337 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/EndSessionHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/EndSessionHandler.java @@ -14,7 +14,7 @@ public class EndSessionHandler implements GetRestHandler { public static final String ID_NAME = "id"; - public static final String ROUTE = "session/end/{" + ID_NAME +"}"; + public static final String ROUTE = "session/end/{" + ID_NAME + "}"; @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/ValidateSessionHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/ValidateSessionHandler.java index bd58409ba..33f1585c1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/ValidateSessionHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/session/ValidateSessionHandler.java @@ -15,7 +15,7 @@ public class ValidateSessionHandler implements GetRestHandler { private final static String ID_NAME = "id"; - public static final String ROUTE = "session/validate/{" + ID_NAME+"}"; + public static final String ROUTE = "session/validate/{" + ID_NAME + "}"; public static SessionState validateSession(String sessionId) { if (sessionId == null) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java index e4e3e6b13..f2314e1b0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java @@ -24,7 +24,7 @@ public class FindTagsGetHandler implements GetRestHandler { public static final String ATTRIBUTE_NAME = "attribute"; public static final String VALUE_NAME = "value"; - public static final String ROUTE = "find/tags/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME+"}"; + public static final String ROUTE = "find/tags/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(FindTagsGetHandler.class); diff --git a/cineast-core/build.gradle b/cineast-core/build.gradle index 3189b85ea..da5136500 100644 --- a/cineast-core/build.gradle +++ b/cineast-core/build.gradle @@ -154,7 +154,7 @@ dependencies { /** JavaCPP. */ api group: "org.bytedeco", name: "javacpp", version: version_javacpp api group: "org.bytedeco.javacpp-presets", name: "ffmpeg", version: version_ffmpeg - + /** OpenCV. */ api group: 'org.openpnp', name: 'opencv', version: version_opencv @@ -166,7 +166,7 @@ dependencies { api group: 'javax.activation', name: 'activation', version: '1.1.1' } -shadowJar{ +shadowJar { mergeServiceFiles() } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ColorConverter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ColorConverter.java index 0a3aeebb6..d21bc7ac9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ColorConverter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ColorConverter.java @@ -316,9 +316,7 @@ public static ReadableHSVContainer cachedRGBtoHSV(int rgb) { } /** - * @param colors assumed to be RGB and (x,y) can be accessed at y*width+x TODO Mabye use - * https://stackoverflow.com/questions/596216/formula-to-determine-brightness-of-rgb-color for - * optimization + * @param colors assumed to be RGB and (x,y) can be accessed at y*width+x TODO Mabye use https://stackoverflow.com/questions/596216/formula-to-determine-brightness-of-rgb-color for optimization */ public static float[] RGBtoLuminance(int[] colors) { float[] luminance = new float[colors.length]; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/FuzzyColorHistogramQuantizer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/FuzzyColorHistogramQuantizer.java index 8d3e99bdc..3f8d0f47b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/FuzzyColorHistogramQuantizer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/FuzzyColorHistogramQuantizer.java @@ -6,251 +6,253 @@ /* based on Article "Fuzzy color histogram-based video segmentation" by Küçüktunç, Onur, Uğur Güdükbay, and Özgür Ulusoy */ public class FuzzyColorHistogramQuantizer { - private static final Logger LOGGER = LogManager.getLogger(); - - private FuzzyColorHistogramQuantizer(){} - - public static enum Color{ - Black(new RGBContainer(0, 0, 0)), - Blue(new RGBContainer(0, 0, 255)), - Navy(new RGBContainer(0, 0, 128)), - Red(new RGBContainer(255, 0, 0)), - Yellow(new RGBContainer(255, 255, 0)), - Magenta(new RGBContainer(255, 0, 255)), - Brown(new RGBContainer(150, 75, 0)), - Grey(new RGBContainer(128, 128, 128)), - Green(new RGBContainer(0, 255, 0)), - Teal(new RGBContainer(0, 128, 128)), - Violet(new RGBContainer(143, 0, 255)), - Orange(new RGBContainer(255, 127, 0)), - Pink(new RGBContainer(255, 192, 203)), - White(new RGBContainer(255, 255, 255)), - Cyan(new RGBContainer(0, 255, 255)); - - private final ReadableRGBContainer rgbc; - - Color(ReadableRGBContainer rgbc){ - this.rgbc = rgbc; - } - - public ReadableRGBContainer getRGB(){ - return this.rgbc; - } - - }; - - static boolean isBlack(ReadableLabContainer lab){ - return lab.L < 40f; - } - - static boolean isGrey(ReadableLabContainer lab){ - return lab.L > 10f && lab.L < 90f; - } - - static boolean isWhite(ReadableLabContainer lab){ - return lab.L > 60f; - } - - static boolean isGreen(ReadableLabContainer lab){ - return lab.a < -30f; - } - - static boolean isGreenish(ReadableLabContainer lab){ - return lab.a > -70f && lab.a < 0f; - } - - static boolean isAmiddle(ReadableLabContainer lab){ - return lab.a > -12f && lab.a < 12f; - } - - static boolean isReddish(ReadableLabContainer lab){ - return lab.a > 0f && lab.a < 80f; - } - - static boolean isRed(ReadableLabContainer lab){ - return lab.a > 40f; - } - - static boolean isBlue(ReadableLabContainer lab){ - return lab.b < -50f; - } - - static boolean isBluish(ReadableLabContainer lab){ - return lab.b > -90f && lab.b < 0f; - } - - static boolean isBmiddle(ReadableLabContainer lab){ - return lab.b > -12f && lab.b < 12f; - } - - static boolean isYellowish(ReadableLabContainer lab){ - return lab.b > 0f && lab.b < 75f; - } - - static boolean isYellow(ReadableLabContainer lab){ - return lab.b > 35f; - } - - public static Color quantize(ReadableLabContainer lab){ - - if(isWhite(lab)){ - - if(isAmiddle(lab)){ - if(isBmiddle(lab)){ - return Color.White; - } - if(isYellowish(lab) || isYellow(lab)){ - return Color.Yellow; - } - return Color.Blue; - } - - if(isReddish(lab)){ - if(isBluish(lab)){ - return Color.Magenta; - } - if(isYellow(lab)){ - return Color.Brown; - } - if(isBmiddle(lab)){ - return Color.Pink; - } - } - - if(isGreenish(lab)){ - if(isBmiddle(lab)){ - return Color.Cyan; - } - if(isBluish(lab)){ - return Color.Blue; - } - if(isYellowish(lab)){ - return Color.Green; - } - if(isYellow(lab)){ - return Color.Yellow; - } - } - - if(isRed(lab)){ - if(isYellow(lab)){ - return Color.Orange; - } - } - - if(isGreen(lab)){ - if(isBlue(lab) || isBluish(lab)){ - return Color.Cyan; - } - if(isYellow(lab) || isYellowish(lab)){ - return Color.Green; - } - } - - } - - if(isBlack(lab)){ - if(isAmiddle(lab)){ - if(isBmiddle(lab)){ - return Color.Black; - } - if(isBlue(lab) || isBluish(lab)){ - return Color.Blue; - } - } - if(isReddish(lab)){ - if(isBluish(lab)){ - return Color.Blue; - } - if(isYellowish(lab)){ - return Color.Red; - } - if(isBlue(lab)){ - return Color.Navy; - } - } - if(isGreenish(lab)){ - if(isBluish(lab)){ - return Color.Blue; - } - if(isYellowish(lab)){ - return Color.Teal; - } - } - if(isRed(lab)){ - if(isBluish(lab)){ - return Color.Blue; - } - } - if(isGreen(lab)){ - if(isBluish(lab)){ - return Color.Blue; - } - } - } - - - if(isGrey(lab)){ - if(isAmiddle(lab)){ - if(isBmiddle(lab)){ - return Color.Grey; - } - if(isBlue(lab)){ - return Color.Blue; - } - if(isYellowish(lab)){ - return Color.Brown; - } - } - if(isReddish(lab)){ - if(isBmiddle(lab)){ - return Color.Red; - } - if(isBluish(lab)){ - return Color.Magenta; - } - if(isYellowish(lab)){ - return Color.Orange; - } - if(isBlue(lab)){ - return Color.Blue; - } - if(isYellow(lab)){ - return Color.Yellow; - } - } - if(isGreenish(lab)){ - if(isBmiddle(lab)){ - return Color.Teal; - } - if(isBlue(lab)){ - return Color.Blue; - } - if(isYellowish(lab)){ - return Color.Green; - } - if(isBluish(lab)){ - return Color.Teal; - } - } - if(isRed(lab)){ - if(isBlue(lab)){ - return Color.Blue; - } - return Color.Red; - } - if(isGreen(lab)){ - if(isBlue(lab)){ - return Color.Violet; - } - if(isYellow(lab)){ - return Color.Green; - } - } - } - - LOGGER.warn("Error while quantizing {} returning Black", lab); - return Color.Black; - } - + private static final Logger LOGGER = LogManager.getLogger(); + + private FuzzyColorHistogramQuantizer() { + } + + public static enum Color { + Black(new RGBContainer(0, 0, 0)), + Blue(new RGBContainer(0, 0, 255)), + Navy(new RGBContainer(0, 0, 128)), + Red(new RGBContainer(255, 0, 0)), + Yellow(new RGBContainer(255, 255, 0)), + Magenta(new RGBContainer(255, 0, 255)), + Brown(new RGBContainer(150, 75, 0)), + Grey(new RGBContainer(128, 128, 128)), + Green(new RGBContainer(0, 255, 0)), + Teal(new RGBContainer(0, 128, 128)), + Violet(new RGBContainer(143, 0, 255)), + Orange(new RGBContainer(255, 127, 0)), + Pink(new RGBContainer(255, 192, 203)), + White(new RGBContainer(255, 255, 255)), + Cyan(new RGBContainer(0, 255, 255)); + + private final ReadableRGBContainer rgbc; + + Color(ReadableRGBContainer rgbc) { + this.rgbc = rgbc; + } + + public ReadableRGBContainer getRGB() { + return this.rgbc; + } + + } + + ; + + static boolean isBlack(ReadableLabContainer lab) { + return lab.L < 40f; + } + + static boolean isGrey(ReadableLabContainer lab) { + return lab.L > 10f && lab.L < 90f; + } + + static boolean isWhite(ReadableLabContainer lab) { + return lab.L > 60f; + } + + static boolean isGreen(ReadableLabContainer lab) { + return lab.a < -30f; + } + + static boolean isGreenish(ReadableLabContainer lab) { + return lab.a > -70f && lab.a < 0f; + } + + static boolean isAmiddle(ReadableLabContainer lab) { + return lab.a > -12f && lab.a < 12f; + } + + static boolean isReddish(ReadableLabContainer lab) { + return lab.a > 0f && lab.a < 80f; + } + + static boolean isRed(ReadableLabContainer lab) { + return lab.a > 40f; + } + + static boolean isBlue(ReadableLabContainer lab) { + return lab.b < -50f; + } + + static boolean isBluish(ReadableLabContainer lab) { + return lab.b > -90f && lab.b < 0f; + } + + static boolean isBmiddle(ReadableLabContainer lab) { + return lab.b > -12f && lab.b < 12f; + } + + static boolean isYellowish(ReadableLabContainer lab) { + return lab.b > 0f && lab.b < 75f; + } + + static boolean isYellow(ReadableLabContainer lab) { + return lab.b > 35f; + } + + public static Color quantize(ReadableLabContainer lab) { + + if (isWhite(lab)) { + + if (isAmiddle(lab)) { + if (isBmiddle(lab)) { + return Color.White; + } + if (isYellowish(lab) || isYellow(lab)) { + return Color.Yellow; + } + return Color.Blue; + } + + if (isReddish(lab)) { + if (isBluish(lab)) { + return Color.Magenta; + } + if (isYellow(lab)) { + return Color.Brown; + } + if (isBmiddle(lab)) { + return Color.Pink; + } + } + + if (isGreenish(lab)) { + if (isBmiddle(lab)) { + return Color.Cyan; + } + if (isBluish(lab)) { + return Color.Blue; + } + if (isYellowish(lab)) { + return Color.Green; + } + if (isYellow(lab)) { + return Color.Yellow; + } + } + + if (isRed(lab)) { + if (isYellow(lab)) { + return Color.Orange; + } + } + + if (isGreen(lab)) { + if (isBlue(lab) || isBluish(lab)) { + return Color.Cyan; + } + if (isYellow(lab) || isYellowish(lab)) { + return Color.Green; + } + } + + } + + if (isBlack(lab)) { + if (isAmiddle(lab)) { + if (isBmiddle(lab)) { + return Color.Black; + } + if (isBlue(lab) || isBluish(lab)) { + return Color.Blue; + } + } + if (isReddish(lab)) { + if (isBluish(lab)) { + return Color.Blue; + } + if (isYellowish(lab)) { + return Color.Red; + } + if (isBlue(lab)) { + return Color.Navy; + } + } + if (isGreenish(lab)) { + if (isBluish(lab)) { + return Color.Blue; + } + if (isYellowish(lab)) { + return Color.Teal; + } + } + if (isRed(lab)) { + if (isBluish(lab)) { + return Color.Blue; + } + } + if (isGreen(lab)) { + if (isBluish(lab)) { + return Color.Blue; + } + } + } + + if (isGrey(lab)) { + if (isAmiddle(lab)) { + if (isBmiddle(lab)) { + return Color.Grey; + } + if (isBlue(lab)) { + return Color.Blue; + } + if (isYellowish(lab)) { + return Color.Brown; + } + } + if (isReddish(lab)) { + if (isBmiddle(lab)) { + return Color.Red; + } + if (isBluish(lab)) { + return Color.Magenta; + } + if (isYellowish(lab)) { + return Color.Orange; + } + if (isBlue(lab)) { + return Color.Blue; + } + if (isYellow(lab)) { + return Color.Yellow; + } + } + if (isGreenish(lab)) { + if (isBmiddle(lab)) { + return Color.Teal; + } + if (isBlue(lab)) { + return Color.Blue; + } + if (isYellowish(lab)) { + return Color.Green; + } + if (isBluish(lab)) { + return Color.Teal; + } + } + if (isRed(lab)) { + if (isBlue(lab)) { + return Color.Blue; + } + return Color.Red; + } + if (isGreen(lab)) { + if (isBlue(lab)) { + return Color.Violet; + } + if (isYellow(lab)) { + return Color.Green; + } + } + } + + LOGGER.warn("Error while quantizing {} returning Black", lab); + return Color.Black; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/HSVContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/HSVContainer.java index 7a0fbd9ae..0406cfc67 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/HSVContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/HSVContainer.java @@ -2,28 +2,37 @@ import org.vitrivr.cineast.core.data.FloatVector; -public class HSVContainer extends ReadableHSVContainer implements FloatVector{ - - public HSVContainer(ReadableHSVContainer hsv){ - super(hsv.h, hsv.s, hsv.v); - } - - public HSVContainer(float h, float s, float v) { - super(h, s, v); - } - - public HSVContainer(double h, double s, double v) { - super(h, s, v); - } - - @Override - public void setElement(int num, float val) { - switch(num){ - case 0:{h = val; break;} - case 1:{s = val; break;} - case 2:{v = val; break;} - } - - } +public class HSVContainer extends ReadableHSVContainer implements FloatVector { + + public HSVContainer(ReadableHSVContainer hsv) { + super(hsv.h, hsv.s, hsv.v); + } + + public HSVContainer(float h, float s, float v) { + super(h, s, v); + } + + public HSVContainer(double h, double s, double v) { + super(h, s, v); + } + + @Override + public void setElement(int num, float val) { + switch (num) { + case 0: { + h = val; + break; + } + case 1: { + s = val; + break; + } + case 2: { + v = val; + break; + } + } + + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/LabContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/LabContainer.java index 47146d2aa..c28c0aeeb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/LabContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/LabContainer.java @@ -2,55 +2,67 @@ import org.vitrivr.cineast.core.data.FloatVector; -public class LabContainer extends ReadableLabContainer implements FloatVector{ - - public LabContainer(){ - super(0, 0, 0); - } - - public LabContainer(double L, double a, double b) { - super(L, a, b); - } - - public LabContainer(double L, double a, double b, double alpha) { - super(L, a, b, alpha); - } - - public LabContainer(float L, float a, float b) { - super(L, a, b); - } - - public LabContainer(float L, float a, float b, float alpha) { - super(L, a, b, alpha); - } - - public LabContainer(ReadableLabContainer lab){ - super(lab.L, lab.a, lab.b, lab.alpha); - } - - public void setL(float L){ - this.L = L; - } - - public void setA(float a){ - this.a = a; - } - - public void setB(float b){ - this.b = b; - } - - public void setAlpha(float alpha){ - this.alpha = alpha; - } - - @Override - public void setElement(int num, float val) { - switch(num){ - case 0:{L = val; break;} - case 1:{a = val; break;} - case 2:{b = val; break;} - case 3:{alpha = val; break;} - } - } +public class LabContainer extends ReadableLabContainer implements FloatVector { + + public LabContainer() { + super(0, 0, 0); + } + + public LabContainer(double L, double a, double b) { + super(L, a, b); + } + + public LabContainer(double L, double a, double b, double alpha) { + super(L, a, b, alpha); + } + + public LabContainer(float L, float a, float b) { + super(L, a, b); + } + + public LabContainer(float L, float a, float b, float alpha) { + super(L, a, b, alpha); + } + + public LabContainer(ReadableLabContainer lab) { + super(lab.L, lab.a, lab.b, lab.alpha); + } + + public void setL(float L) { + this.L = L; + } + + public void setA(float a) { + this.a = a; + } + + public void setB(float b) { + this.b = b; + } + + public void setAlpha(float alpha) { + this.alpha = alpha; + } + + @Override + public void setElement(int num, float val) { + switch (num) { + case 0: { + L = val; + break; + } + case 1: { + a = val; + break; + } + case 2: { + b = val; + break; + } + case 3: { + alpha = val; + break; + } + } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/RGBContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/RGBContainer.java index 5fd27cb93..98a603a0b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/RGBContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/RGBContainer.java @@ -2,101 +2,112 @@ import org.vitrivr.cineast.core.data.FloatVector; -public class RGBContainer extends ReadableRGBContainer implements FloatVector{ - - public RGBContainer(ReadableRGBContainer rgb){ - super(rgb.r, rgb.g, rgb.b); - } - - RGBContainer() { - super(0, 0, 0); - } - - public RGBContainer(int r, int g, int b){ - super(r, g, b); - } - - public RGBContainer(int r, int g, int b, int a){ - super(r, g, b, a); - } - - public RGBContainer(float r, float g, float b){ - super(r, g, b); - } - - public RGBContainer(float r, float g, float b, float a){ - super(r, g, b, a); - } - - public RGBContainer(double r, double g, double b){ - super(r, g, b); - } - - public RGBContainer(double r, double g, double b, double a){ - super(r, g, b, a); - } - - public RGBContainer(int color){ - super(color); - } - - @Override - public void setElement(int num, float val) { - switch(num){ - case 0:{r = Math.round(val * 255f); break;} - case 1:{g = Math.round(val * 255f); break;} - case 2:{b = Math.round(val * 255f); break;} - case 3:{a = Math.round(val * 255f); break;} - } - - } - - - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + a; - result = prime * result + b; - result = prime * result + g; - result = prime * result + r; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { +public class RGBContainer extends ReadableRGBContainer implements FloatVector { + + public RGBContainer(ReadableRGBContainer rgb) { + super(rgb.r, rgb.g, rgb.b); + } + + RGBContainer() { + super(0, 0, 0); + } + + public RGBContainer(int r, int g, int b) { + super(r, g, b); + } + + public RGBContainer(int r, int g, int b, int a) { + super(r, g, b, a); + } + + public RGBContainer(float r, float g, float b) { + super(r, g, b); + } + + public RGBContainer(float r, float g, float b, float a) { + super(r, g, b, a); + } + + public RGBContainer(double r, double g, double b) { + super(r, g, b); + } + + public RGBContainer(double r, double g, double b, double a) { + super(r, g, b, a); + } + + public RGBContainer(int color) { + super(color); + } + + @Override + public void setElement(int num, float val) { + switch (num) { + case 0: { + r = Math.round(val * 255f); + break; + } + case 1: { + g = Math.round(val * 255f); + break; + } + case 2: { + b = Math.round(val * 255f); + break; + } + case 3: { + a = Math.round(val * 255f); + break; + } + } + + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + a; + result = prime * result + b; + result = prime * result + g; + result = prime * result + r; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - RGBContainer other = (RGBContainer) obj; - if (a != other.a) { + RGBContainer other = (RGBContainer) obj; + if (a != other.a) { return false; } - if (b != other.b) { + if (b != other.b) { return false; } - if (g != other.g) { + if (g != other.g) { return false; } - if (r != other.r) { + if (r != other.r) { return false; } - return true; - } - - public RGBContainer set(int intcolor){ - this.r = getRed(intcolor); - this.g = getGreen(intcolor); - this.b = getBlue(intcolor); - this.a = getAlpha(intcolor); - return this; - } - + return true; + } + + public RGBContainer set(int intcolor) { + this.r = getRed(intcolor); + this.g = getGreen(intcolor); + this.b = getBlue(intcolor); + this.a = getAlpha(intcolor); + return this; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableHSVContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableHSVContainer.java index 56e050a27..3e557fe55 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableHSVContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableHSVContainer.java @@ -18,10 +18,14 @@ public ReadableHSVContainer(double h, double s, double v) { @Override public float getElement(int num) { switch (num) { - case 0: return h; - case 1: return s; - case 2: return v; - default: throw new IndexOutOfBoundsException(num + ">= 3"); + case 0: + return h; + case 1: + return s; + case 2: + return v; + default: + throw new IndexOutOfBoundsException(num + ">= 3"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableLabContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableLabContainer.java index 06b25da73..df2de684b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableLabContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableLabContainer.java @@ -70,8 +70,7 @@ public float getChroma() { } /** - * see Lübbe, Eva (2010). Colours in the Mind - Colour Systems in Reality- A formula for colour - * saturation + * see Lübbe, Eva (2010). Colours in the Mind - Colour Systems in Reality- A formula for colour saturation * * @return the saturation */ @@ -90,11 +89,16 @@ public float getHue() { @Override public float getElement(int num) { switch (num) { - case 0: return L; - case 1: return a; - case 2: return b; - case 3: return alpha; - default: throw new IndexOutOfBoundsException(num + ">= 4"); + case 0: + return L; + case 1: + return a; + case 2: + return b; + case 3: + return alpha; + default: + throw new IndexOutOfBoundsException(num + ">= 4"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableRGBContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableRGBContainer.java index cce0629a1..de8f53113 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableRGBContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableRGBContainer.java @@ -127,24 +127,27 @@ public String toFeatureString() { return "<" + r + ", " + g + ", " + b + ">"; } - public static ReadableRGBContainer fromColorString(String colorString){ - if (colorString == null || colorString.length() != 7){ + public static ReadableRGBContainer fromColorString(String colorString) { + if (colorString == null || colorString.length() != 7) { return new ReadableRGBContainer(0); } int r = 0, g = 0, b = 0; - try{ - r = Integer.parseInt(colorString.substring(1,3), 16); - }catch (NumberFormatException e){} + try { + r = Integer.parseInt(colorString.substring(1, 3), 16); + } catch (NumberFormatException e) { + } - try{ - g = Integer.parseInt(colorString.substring(3,5), 16); - }catch (NumberFormatException e){} + try { + g = Integer.parseInt(colorString.substring(3, 5), 16); + } catch (NumberFormatException e) { + } - try{ - b = Integer.parseInt(colorString.substring(5,7), 16); - }catch (NumberFormatException e){} + try { + b = Integer.parseInt(colorString.substring(5, 7), 16); + } catch (NumberFormatException e) { + } return new ReadableRGBContainer(r, g, b); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableXYZContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableXYZContainer.java index e83409975..957fa3f25 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableXYZContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableXYZContainer.java @@ -23,10 +23,14 @@ public String toString() { @Override public float getElement(int num) { switch (num) { - case 0: return x; - case 1: return y; - case 2: return z; - default: throw new IndexOutOfBoundsException(num + ">= 3"); + case 0: + return x; + case 1: + return y; + case 2: + return z; + default: + throw new IndexOutOfBoundsException(num + ">= 3"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableYCbCrContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableYCbCrContainer.java index 7b4015e0f..eb9ca72e8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableYCbCrContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/ReadableYCbCrContainer.java @@ -14,10 +14,14 @@ public ReadableYCbCrContainer(int Y, int Cb, int Cr) { @Override public float getElement(int num) { switch (num) { - case 0: return y / 255f; - case 1: return cb / 255f; - case 2: return cr / 255f; - default: throw new IndexOutOfBoundsException(num + ">= 3"); + case 0: + return y / 255f; + case 1: + return cb / 255f; + case 2: + return cr / 255f; + default: + throw new IndexOutOfBoundsException(num + ">= 3"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/XYZContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/XYZContainer.java index d527f8799..edb0e5c3a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/XYZContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/XYZContainer.java @@ -2,29 +2,38 @@ import org.vitrivr.cineast.core.data.FloatVector; -public class XYZContainer extends ReadableXYZContainer implements FloatVector{ - - public XYZContainer(float x, float y, float z){ - super(x, y, z); - } - - public XYZContainer(double x, double y, double z){ - super(x, y, z); - } - - XYZContainer() { - this(0f, 0f, 0f); - } - - - @Override - public void setElement(int num, float val) { - switch(num){ - case 0:{x = val; break;} - case 1:{y = val; break;} - case 2:{z = val; break;} - } - } - - +public class XYZContainer extends ReadableXYZContainer implements FloatVector { + + public XYZContainer(float x, float y, float z) { + super(x, y, z); + } + + public XYZContainer(double x, double y, double z) { + super(x, y, z); + } + + XYZContainer() { + this(0f, 0f, 0f); + } + + + @Override + public void setElement(int num, float val) { + switch (num) { + case 0: { + x = val; + break; + } + case 1: { + y = val; + break; + } + case 2: { + z = val; + break; + } + } + } + + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/YCbCrContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/YCbCrContainer.java index 578c32385..473b44fde 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/color/YCbCrContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/color/YCbCrContainer.java @@ -2,25 +2,34 @@ import org.vitrivr.cineast.core.data.FloatVector; -public class YCbCrContainer extends ReadableYCbCrContainer implements FloatVector{ - - public YCbCrContainer(ReadableYCbCrContainer ycbcr){ - super(ycbcr.y, ycbcr.cb, ycbcr.cr); - } - - public YCbCrContainer(int Y, int Cb, int Cr){ - super(Y, Cb, Cr); - } - - - @Override - public void setElement(int num, float val) { - switch(num){ - case 0:{y = Math.round(val * 255f); break;} - case 1:{cb = Math.round(val * 255f); break;} - case 2:{cr = Math.round(val * 255f); break;} - } - } +public class YCbCrContainer extends ReadableYCbCrContainer implements FloatVector { + + public YCbCrContainer(ReadableYCbCrContainer ycbcr) { + super(ycbcr.y, ycbcr.cb, ycbcr.cr); + } + + public YCbCrContainer(int Y, int Cb, int Cr) { + super(Y, Cb, Cr); + } + + + @Override + public void setElement(int num, float val) { + switch (num) { + case 0: { + y = Math.round(val * 255f); + break; + } + case 1: { + cb = Math.round(val * 255f); + break; + } + case 2: { + cr = Math.round(val * 255f); + break; + } + } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/CacheConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/CacheConfig.java index 985de9015..24bd85aac 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/CacheConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/CacheConfig.java @@ -2,142 +2,143 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.UUID; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; public final class CacheConfig { - public enum Policy{ - DISK_CACHE, //cache to disk unless newInMemoryMultiImage is requested - AUTOMATIC, //keep in memory as long as threshold is not exceeded, makes exceptions for images not generated by the video decoder - AVOID_CACHE //avoids cache until hard limit is reached - } - - private static final Logger LOGGER = LogManager.getLogger(); - - private Policy cachingPolicy = Policy.AUTOMATIC; - private Path cacheLocation = Paths.get("."); - private final UUID uuid = UUID.randomUUID(); - private CachedDataFactory factory; - - public CacheConfig() {} - - /** - * - * @param cachePolicy Caching Policy - * @param cacheLocation the file system location of the disk cache - * @throws IllegalArgumentException in case any of the memory limits is negative - * @throws NullPointerException in case the cachePolicy or cacheLocation is null - * @throws SecurityException in case access to cacheLocation is not permitted - */ - @JsonCreator - public CacheConfig( - @JsonProperty(value = "cachePolicy", required = false, defaultValue = "AUTOMATIC") String cachePolicy, - @JsonProperty(value = "cacheLocation", required = false, defaultValue = ".") String cacheLocation){ - - if (cachePolicy == null) { - cachePolicy = "AUTOMATIC"; - } - if (cacheLocation == null) { - cacheLocation = "."; - } - - final Path location = Paths.get(cacheLocation); - if (!Files.exists(location)) { - try { - Files.createDirectories(location); - this.cacheLocation = location; - } catch (IOException e) { - this.cacheLocation = Paths.get("."); - LOGGER.warn("Specified cache location ({}) could not be created! Fallback to default location: {}", location.toAbsolutePath().toString(), this.cacheLocation.toAbsolutePath().toString()); - } - } else if (!Files.isDirectory(location)) { - this.cacheLocation = Paths.get("."); - LOGGER.warn("Specified cache location ({}) could not be created! Fallback to default location: {}", location.toAbsolutePath().toString(), this.cacheLocation.toAbsolutePath().toString()); - } else { - this.cacheLocation = location; - } - this.cachingPolicy = Policy.valueOf(cachePolicy); - } - - /** - * @return the caching policy - */ - @JsonProperty - public final Policy getCachingPolicy(){ - return this.cachingPolicy; - } - public void setCachingPolicy(Policy cachingPolicy) { - if(cachingPolicy == null){ - throw new NullPointerException("CachePolicy cannot be null"); - } - this.cachingPolicy = cachingPolicy; - } - - /** - * @return the file system location of the cache - */ - @JsonProperty - public final Path getCacheLocation(){ - return this.cacheLocation; - } - public void setCacheLocation(Path cacheLocation) { - if(cacheLocation == null){ - throw new NullPointerException("CacheLocation cannot be null"); - } - this.cacheLocation = cacheLocation; - } - - /** - * Returns the UUID of this {@link CacheConfig}. - * - * @return UUID of this {@link CacheConfig}. - */ - public String getUUID() { - return this.uuid.toString(); - } - - /** - * A simple heuristic to determine whether an object of the given size should be cached or kept in-memory. - * - * @param size Size of the object in bytes. - * @return True if object should be kept in memory, false otherwise. - */ - public boolean keepInMemory(int size) { - final double utilisation = ((double)(Runtime.getRuntime().freeMemory() + size + 24)/(double)Runtime.getRuntime().maxMemory()); - switch (this.cachingPolicy) { - case DISK_CACHE: - return false; - case AUTOMATIC: - return utilisation > 0.65; /* If more than 65% of memory is occupied, we start caching. */ - case AVOID_CACHE: - return size > 0.90; /* If more than 90% of memory is occupied, we start caching. */ - default: - return true; - } - } - - /** - * Returns and optionally creates the shared {@link CachedDataFactory} instance created by this {@link CacheConfig}. - * - * @return Shared {@link CachedDataFactory}. - */ - public synchronized CachedDataFactory sharedCachedDataFactory() { - if (this.factory == null) { - this.factory = new CachedDataFactory(this); - } - return this.factory; - } - - @Override - public String toString(){ - return "\"cache\" : { \"cachePolicy\" : \"" + this.cachingPolicy.toString() + ", \"cacheLocation\" : \"" + this.cacheLocation.toString() + "\" }"; - } + + public enum Policy { + DISK_CACHE, //cache to disk unless newInMemoryMultiImage is requested + AUTOMATIC, //keep in memory as long as threshold is not exceeded, makes exceptions for images not generated by the video decoder + AVOID_CACHE //avoids cache until hard limit is reached + } + + private static final Logger LOGGER = LogManager.getLogger(); + + private Policy cachingPolicy = Policy.AUTOMATIC; + private Path cacheLocation = Paths.get("."); + private final UUID uuid = UUID.randomUUID(); + private CachedDataFactory factory; + + public CacheConfig() { + } + + /** + * @param cachePolicy Caching Policy + * @param cacheLocation the file system location of the disk cache + * @throws IllegalArgumentException in case any of the memory limits is negative + * @throws NullPointerException in case the cachePolicy or cacheLocation is null + * @throws SecurityException in case access to cacheLocation is not permitted + */ + @JsonCreator + public CacheConfig( + @JsonProperty(value = "cachePolicy", required = false, defaultValue = "AUTOMATIC") String cachePolicy, + @JsonProperty(value = "cacheLocation", required = false, defaultValue = ".") String cacheLocation) { + + if (cachePolicy == null) { + cachePolicy = "AUTOMATIC"; + } + if (cacheLocation == null) { + cacheLocation = "."; + } + + final Path location = Paths.get(cacheLocation); + if (!Files.exists(location)) { + try { + Files.createDirectories(location); + this.cacheLocation = location; + } catch (IOException e) { + this.cacheLocation = Paths.get("."); + LOGGER.warn("Specified cache location ({}) could not be created! Fallback to default location: {}", location.toAbsolutePath().toString(), this.cacheLocation.toAbsolutePath().toString()); + } + } else if (!Files.isDirectory(location)) { + this.cacheLocation = Paths.get("."); + LOGGER.warn("Specified cache location ({}) could not be created! Fallback to default location: {}", location.toAbsolutePath().toString(), this.cacheLocation.toAbsolutePath().toString()); + } else { + this.cacheLocation = location; + } + this.cachingPolicy = Policy.valueOf(cachePolicy); + } + + /** + * @return the caching policy + */ + @JsonProperty + public final Policy getCachingPolicy() { + return this.cachingPolicy; + } + + public void setCachingPolicy(Policy cachingPolicy) { + if (cachingPolicy == null) { + throw new NullPointerException("CachePolicy cannot be null"); + } + this.cachingPolicy = cachingPolicy; + } + + /** + * @return the file system location of the cache + */ + @JsonProperty + public final Path getCacheLocation() { + return this.cacheLocation; + } + + public void setCacheLocation(Path cacheLocation) { + if (cacheLocation == null) { + throw new NullPointerException("CacheLocation cannot be null"); + } + this.cacheLocation = cacheLocation; + } + + /** + * Returns the UUID of this {@link CacheConfig}. + * + * @return UUID of this {@link CacheConfig}. + */ + public String getUUID() { + return this.uuid.toString(); + } + + /** + * A simple heuristic to determine whether an object of the given size should be cached or kept in-memory. + * + * @param size Size of the object in bytes. + * @return True if object should be kept in memory, false otherwise. + */ + public boolean keepInMemory(int size) { + final double utilisation = ((double) (Runtime.getRuntime().freeMemory() + size + 24) / (double) Runtime.getRuntime().maxMemory()); + switch (this.cachingPolicy) { + case DISK_CACHE: + return false; + case AUTOMATIC: + return utilisation > 0.65; /* If more than 65% of memory is occupied, we start caching. */ + case AVOID_CACHE: + return size > 0.90; /* If more than 90% of memory is occupied, we start caching. */ + default: + return true; + } + } + + /** + * Returns and optionally creates the shared {@link CachedDataFactory} instance created by this {@link CacheConfig}. + * + * @return Shared {@link CachedDataFactory}. + */ + public synchronized CachedDataFactory sharedCachedDataFactory() { + if (this.factory == null) { + this.factory = new CachedDataFactory(this); + } + return this.factory; + } + + @Override + public String toString() { + return "\"cache\" : { \"cachePolicy\" : \"" + this.cachingPolicy.toString() + ", \"cacheLocation\" : \"" + this.cacheLocation.toString() + "\" }"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java index 976bdd7fb..3c49a0c2e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java @@ -2,11 +2,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.File; +import java.util.function.Supplier; import org.vitrivr.cineast.core.db.DBSelectorSupplier; import org.vitrivr.cineast.core.db.NoDBSelector; import org.vitrivr.cineast.core.db.NoDBWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.adampro.*; +import org.vitrivr.cineast.core.db.adampro.ADAMproEntityCreator; +import org.vitrivr.cineast.core.db.adampro.ADAMproSelector; +import org.vitrivr.cineast.core.db.adampro.ADAMproStreamingSelector; +import org.vitrivr.cineast.core.db.adampro.ADAMproWrapper; +import org.vitrivr.cineast.core.db.adampro.ADAMproWriter; import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; @@ -18,9 +24,6 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.db.setup.NoEntityCreator; -import java.io.File; -import java.util.function.Supplier; - public final class DatabaseConfig { @@ -67,18 +70,18 @@ public enum Selector { private ADAMproWrapper adaMproWrapper = null; - private synchronized void ensureAdamProWrapper(){ - if (this.adaMproWrapper == null){ - this.adaMproWrapper = new ADAMproWrapper(this); - } + private synchronized void ensureAdamProWrapper() { + if (this.adaMproWrapper == null) { + this.adaMproWrapper = new ADAMproWrapper(this); + } } private CottontailWrapper cottontailWrapper = null; - private synchronized void ensureCottontailWrapper(){ - if (this.cottontailWrapper == null){ - this.cottontailWrapper = new CottontailWrapper(this, true); - } + private synchronized void ensureCottontailWrapper() { + if (this.cottontailWrapper == null) { + this.cottontailWrapper = new CottontailWrapper(this, true); + } } @JsonCreator @@ -150,22 +153,22 @@ public synchronized PersistencyWriterSupplier getWriterSupplier() { switch (this.writer) { case NONE: return NO_WRITER_SUPPLY; - case ADAMPRO:{ - if(SINGLE_CONNECTION){ - ensureAdamProWrapper(); - return () -> new ADAMproWriter(this.adaMproWrapper); - } - return () -> new ADAMproWriter(new ADAMproWrapper(this)); + case ADAMPRO: { + if (SINGLE_CONNECTION) { + ensureAdamProWrapper(); + return () -> new ADAMproWriter(this.adaMproWrapper); + } + return () -> new ADAMproWriter(new ADAMproWrapper(this)); } - case JSON:{ - return () -> new JsonFileWriter(new File(this.host)); + case JSON: { + return () -> new JsonFileWriter(new File(this.host)); } case COTTONTAIL: { - if (SINGLE_CONNECTION){ - ensureCottontailWrapper(); - return () -> new CottontailWriter(this.cottontailWrapper); - } - return () -> new CottontailWriter(new CottontailWrapper(this, false)); + if (SINGLE_CONNECTION) { + ensureCottontailWrapper(); + return () -> new CottontailWriter(this.cottontailWrapper); + } + return () -> new CottontailWriter(new CottontailWrapper(this, false)); } case INMEMORY: { return InMemoryWriter::new; @@ -178,31 +181,31 @@ public synchronized PersistencyWriterSupplier getWriterSupplier() { public synchronized DBSelectorSupplier getSelectorSupplier() { switch (this.selector) { - case ADAMPRO:{ - if (SINGLE_CONNECTION){ - ensureAdamProWrapper(); - return () -> new ADAMproSelector(this.adaMproWrapper); - } - return () -> new ADAMproSelector(new ADAMproWrapper(this)); + case ADAMPRO: { + if (SINGLE_CONNECTION) { + ensureAdamProWrapper(); + return () -> new ADAMproSelector(this.adaMproWrapper); + } + return () -> new ADAMproSelector(new ADAMproWrapper(this)); } - case ADAMPROSTREAM:{ - if (SINGLE_CONNECTION){ - ensureAdamProWrapper(); - return () -> new ADAMproStreamingSelector(this.adaMproWrapper); - } - return () -> new ADAMproStreamingSelector(new ADAMproWrapper(this)); + case ADAMPROSTREAM: { + if (SINGLE_CONNECTION) { + ensureAdamProWrapper(); + return () -> new ADAMproStreamingSelector(this.adaMproWrapper); + } + return () -> new ADAMproStreamingSelector(new ADAMproWrapper(this)); } case JSON: { - return () -> new JsonSelector(new File(this.host)); + return () -> new JsonSelector(new File(this.host)); } case NONE: return NO_SELECTOR_SUPPLY; - case COTTONTAIL:{ - if (SINGLE_CONNECTION){ - ensureCottontailWrapper(); - return () -> new CottontailSelector(this.cottontailWrapper); - } - return () -> new CottontailSelector(new CottontailWrapper(this, false)); + case COTTONTAIL: { + if (SINGLE_CONNECTION) { + ensureCottontailWrapper(); + return () -> new CottontailSelector(this.cottontailWrapper); + } + return () -> new CottontailSelector(new CottontailWrapper(this, false)); } default: throw new IllegalStateException("No supplier for selector " + this.selector); @@ -213,21 +216,21 @@ public synchronized DBSelectorSupplier getSelectorSupplier() { public synchronized Supplier getEntityCreatorSupplier() { switch (this.selector) { case ADAMPRO: - case ADAMPROSTREAM:{ - if (SINGLE_CONNECTION){ - ensureAdamProWrapper(); - return () -> new ADAMproEntityCreator(this.adaMproWrapper); - } - return () -> new ADAMproEntityCreator(new ADAMproWrapper(this)); + case ADAMPROSTREAM: { + if (SINGLE_CONNECTION) { + ensureAdamProWrapper(); + return () -> new ADAMproEntityCreator(this.adaMproWrapper); + } + return () -> new ADAMproEntityCreator(new ADAMproWrapper(this)); } case NONE: return NO_CREATOR_SUPPLY; - case COTTONTAIL:{ - if (SINGLE_CONNECTION){ - ensureCottontailWrapper(); - return () -> new CottontailEntityCreator(this.cottontailWrapper); - } - return () -> new CottontailEntityCreator(new CottontailWrapper(this, false)); + case COTTONTAIL: { + if (SINGLE_CONNECTION) { + ensureCottontailWrapper(); + return () -> new CottontailEntityCreator(this.cottontailWrapper); + } + return () -> new CottontailEntityCreator(new CottontailWrapper(this, false)); } case INMEMORY: return InMemoryEntityCreator::new; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DecoderConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DecoderConfig.java index ce2d5933f..c6c0a05f6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DecoderConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DecoderConfig.java @@ -2,91 +2,89 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.HashMap; public final class DecoderConfig { - private String decoder = null; - private HashMap properties = new HashMap<>(); + private String decoder = null; + private HashMap properties = new HashMap<>(); + + @JsonCreator + public DecoderConfig() { + } + + @JsonProperty + public String getDecoder() { + return decoder; + } - @JsonCreator - public DecoderConfig() {} + public void setDecoder(String decoder) { + this.decoder = decoder; + } - @JsonProperty - public String getDecoder() { - return decoder; - } - public void setDecoder(String decoder) { - this.decoder = decoder; - } + @JsonProperty + public HashMap getProperties() { + return properties; + } - @JsonProperty - public HashMap getProperties() { - return properties; - } - public void setProperties(HashMap properties) { - this.properties = properties; - } + public void setProperties(HashMap properties) { + this.properties = properties; + } - /** - * Returns a name decoder-property as Float. If the property is not set, the - * default value passed to the method is returned instead. - * - * @param name Name of the property. - * @param preference Preference value. - * @return Float value of the named property or its preference. - */ - public final Float namedAsFloat(String name, Float preference) { - if (this.properties.containsKey(name)) { - return Float.parseFloat(this.properties.get(name)); - } else { - return preference; - } - } + /** + * Returns a name decoder-property as Float. If the property is not set, the default value passed to the method is returned instead. + * + * @param name Name of the property. + * @param preference Preference value. + * @return Float value of the named property or its preference. + */ + public final Float namedAsFloat(String name, Float preference) { + if (this.properties.containsKey(name)) { + return Float.parseFloat(this.properties.get(name)); + } else { + return preference; + } + } - /** - * Returns a name decoder-property as Integer. If the property is not set, the - * default value passed to the method is returned instead. - * - * @param name Name of the property. - * @param preference Preference value. - * @return Integer value of the named property or its preference. - */ - public Integer namedAsInt(String name, Integer preference) { - if (this.properties.containsKey(name)) { - return Integer.parseInt(this.properties.get(name)); - } else { - return preference; - } - } + /** + * Returns a name decoder-property as Integer. If the property is not set, the default value passed to the method is returned instead. + * + * @param name Name of the property. + * @param preference Preference value. + * @return Integer value of the named property or its preference. + */ + public Integer namedAsInt(String name, Integer preference) { + if (this.properties.containsKey(name)) { + return Integer.parseInt(this.properties.get(name)); + } else { + return preference; + } + } - /** - * Returns a name decoder-property as Boolean. If the property is not set, the - * default value passed to the method is returned instead. - * - * @param name Name of the property. - * @param preference Preference value. - * @return Integer value of the named property or its preference. - */ - public Boolean namedAsBoolean(String name, Boolean preference) { - if (this.properties.containsKey(name)) { - return Boolean.parseBoolean(this.properties.get(name)); - } else { - return preference; - } - } + /** + * Returns a name decoder-property as Boolean. If the property is not set, the default value passed to the method is returned instead. + * + * @param name Name of the property. + * @param preference Preference value. + * @return Integer value of the named property or its preference. + */ + public Boolean namedAsBoolean(String name, Boolean preference) { + if (this.properties.containsKey(name)) { + return Boolean.parseBoolean(this.properties.get(name)); + } else { + return preference; + } + } - /** - * Returns a name decoder-property as String. If the property is not set, the - * default value passed to the method is returned instead. - * - * @param name Name of the property. - * @param preference Preference value. - * @return String value of the named property or its preference. - */ - public String namedAsString(String name, String preference) { - return this.properties.getOrDefault(name, preference); - } + /** + * Returns a name decoder-property as String. If the property is not set, the default value passed to the method is returned instead. + * + * @param name Name of the property. + * @param preference Preference value. + * @return String value of the named property or its preference. + */ + public String namedAsString(String name, String preference) { + return this.properties.getOrDefault(name, preference); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java index 230a9c497..06a354056 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java @@ -3,88 +3,85 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.Map; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.HashMap; -import java.util.Map; - public final class IdConfig { - public enum ExistenceCheck { - CHECK_SKIP, - CHECK_PROCEED - } - - /** - * Name of the ObjectIdGenerator. Must correspond to the simple-name or the FQN of the respective class. - * - * @see ObjectIdGenerator - */ - private final String name; - - /** - * Properties that are being used to initialize the ObjectIdGenerator. - * - * @see ObjectIdGenerator - */ - private final Map properties; - - /** - * Determines the 'existence check mode' for objectId's of {@link MediaObjectDescriptor}s, i.e. whether their uniqueness should be explicitly - * checked and what the consequences of the a collision should be. - * - * CHECK_SKIP = Checks the uniqueness of an ID. If it's not unique, that item is skipped. - * CHECK_PROCEED = Checks the uniqueness of an ID. If it's not unique, that item is still processed but no new descriptor is created. - */ - private final ExistenceCheck existenceCheckMode; - - /** - * Constructor for default {@link IdConfig}. - */ - public IdConfig() { - this("UniqueObjectIdGenerator", ExistenceCheck.CHECK_SKIP, new HashMap<>()); - } - - - /** - * Constructor for {@link IdConfig}. Used for deserialization of a {@link IdConfig} instance from a configuration file. - * - * @param name Name of the {@link ObjectIdGenerator} - * @param existenceCheckMode Determines the 'existence check mode' for objectId's of {@link MediaObjectDescriptor}s - */ - @JsonCreator - public IdConfig(@JsonProperty(value = "name", required = true) String name, - @JsonProperty(value = "existenceCheckMode") ExistenceCheck existenceCheckMode, - @JsonProperty(value = "properties") Map properties) { - - this.name = name; - this.existenceCheckMode = (existenceCheckMode == null ? ExistenceCheck.CHECK_SKIP : existenceCheckMode); - this.properties = (properties == null ? new HashMap<>(0) : properties); - } - - - public String getName() { - return this.name; - } - - public Map getProperties() { - return this.properties; - } - - public ExistenceCheck getExistenceCheckMode() { - return this.existenceCheckMode; - } - - /** - * Returns a new {@link ObjectIdGenerator} based on this {@link IdConfig} or null, if the new {@link ObjectIdGenerator} could not be created. - * - * @return New {@link ObjectIdGenerator} or null. - */ - @JsonIgnore - public ObjectIdGenerator getGenerator() { - return ReflectionHelper.newIdGenerator(this.name, this.properties); - } + public enum ExistenceCheck { + CHECK_SKIP, + CHECK_PROCEED + } + + /** + * Name of the ObjectIdGenerator. Must correspond to the simple-name or the FQN of the respective class. + * + * @see ObjectIdGenerator + */ + private final String name; + + /** + * Properties that are being used to initialize the ObjectIdGenerator. + * + * @see ObjectIdGenerator + */ + private final Map properties; + + /** + * Determines the 'existence check mode' for objectId's of {@link MediaObjectDescriptor}s, i.e. whether their uniqueness should be explicitly checked and what the consequences of the a collision should be. + *

    + * CHECK_SKIP = Checks the uniqueness of an ID. If it's not unique, that item is skipped. CHECK_PROCEED = Checks the uniqueness of an ID. If it's not unique, that item is still processed but no new descriptor is created. + */ + private final ExistenceCheck existenceCheckMode; + + /** + * Constructor for default {@link IdConfig}. + */ + public IdConfig() { + this("UniqueObjectIdGenerator", ExistenceCheck.CHECK_SKIP, new HashMap<>()); + } + + + /** + * Constructor for {@link IdConfig}. Used for deserialization of a {@link IdConfig} instance from a configuration file. + * + * @param name Name of the {@link ObjectIdGenerator} + * @param existenceCheckMode Determines the 'existence check mode' for objectId's of {@link MediaObjectDescriptor}s + */ + @JsonCreator + public IdConfig(@JsonProperty(value = "name", required = true) String name, + @JsonProperty(value = "existenceCheckMode") ExistenceCheck existenceCheckMode, + @JsonProperty(value = "properties") Map properties) { + + this.name = name; + this.existenceCheckMode = (existenceCheckMode == null ? ExistenceCheck.CHECK_SKIP : existenceCheckMode); + this.properties = (properties == null ? new HashMap<>(0) : properties); + } + + + public String getName() { + return this.name; + } + + public Map getProperties() { + return this.properties; + } + + public ExistenceCheck getExistenceCheckMode() { + return this.existenceCheckMode; + } + + /** + * Returns a new {@link ObjectIdGenerator} based on this {@link IdConfig} or null, if the new {@link ObjectIdGenerator} could not be created. + * + * @return New {@link ObjectIdGenerator} or null. + */ + @JsonIgnore + public ObjectIdGenerator getGenerator() { + return ReflectionHelper.newIdGenerator(this.name, this.properties); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/QueryConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/QueryConfig.java index c0689dacd..2c2c39ea2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/QueryConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/QueryConfig.java @@ -8,8 +8,6 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; -import org.apache.commons.lang3.builder.ReflectionToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; import org.vitrivr.cineast.core.data.CorrespondenceFunction; public class QueryConfig extends ReadableQueryConfig { @@ -18,7 +16,7 @@ public class QueryConfig extends ReadableQueryConfig { * Constructor used to parse a {@link QueryConfig} from JSON. * * @param queryId The ID of the query. If not set, a new ID will be generated. - * @param hints List of query {@link Hints} used by the query. May be null or empty. + * @param hints List of query {@link Hints} used by the query. May be null or empty. */ @JsonCreator public QueryConfig(@JsonProperty(value = "queryId", required = false) String queryId, diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/ReadableQueryConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/ReadableQueryConfig.java index cd838ec0b..2e9570ff8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/ReadableQueryConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/ReadableQueryConfig.java @@ -2,180 +2,183 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import org.vitrivr.cineast.core.data.CorrespondenceFunction; -import java.util.*; - public class ReadableQueryConfig { - private static final int DEFAULT_RESULTS_PER_MODULE = 250; - - /** - * Possible distance functions that can be configured in the {@link QueryConfig}. It's up to the implementing selector to support these distances and / or provide fallback options. - */ - public enum Distance { - chisquared, correlation, cosine, hamming, jaccard, kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, minkowski, spannorm, haversine - } - - /** - * List of Query-Hints that can be configured in the {@link QueryConfig}. It's up to the implementing selector to actually consider these hints. - */ - public enum Hints { - exact, /* Only exact lookup methods should be considered. */ - inexact, /* Inexact lookup methods can be used as well. */ - lsh, ecp, mi, pq, sh, va, vaf, vav, sequential, empirical - } - - private final UUID queryId; - protected Distance distance = null; - protected float[] distanceWeights = null; - protected float norm = Float.NaN; - protected CorrespondenceFunction correspondence = null; - protected int resultsPerModule = -1; - protected Optional maxResults = Optional.empty(); - protected final Set hints; - protected final Set relevantSegmentIds = new HashSet<>(); - - /** - * Constructor used to parse a {@link ReadableQueryConfig} from JSON. - * - * @param queryId The ID of the query. If not set, a new ID will be generated. - * @param hints List of query {@link Hints} used by the query. May be null or empty. - */ - @JsonCreator - public ReadableQueryConfig(@JsonProperty(value = "queryId", required = false) String queryId, - @JsonProperty(value = "hints", required = false) List hints) { - - - UUID uuid = null; - try { - uuid = UUID.fromString(queryId); - } catch (IllegalArgumentException | NullPointerException e) { - uuid = UUID.randomUUID(); - } - this.queryId = uuid; - if (hints != null) { - this.hints = new HashSet<>(hints); - } else { - this.hints = new HashSet<>(); - } - } - - /** - * Constructor used to create a {@link ReadableQueryConfig} from another {@link ReadableQueryConfig}. - * - * @param qc The {@link ReadableQueryConfig} that should be used as template - */ - public ReadableQueryConfig(ReadableQueryConfig qc) { - this(qc, qc == null ? null : qc.queryId); - } - - /** - * Internal constructor used to create a {@link ReadableQueryConfig} from another {@link ReadableQueryConfig}. - * - * @param qc The {@link ReadableQueryConfig} that should be used. May be null. - * @param uuid The UUID for the new {@link ReadableQueryConfig}. If null, a new UUID will be created. - */ - protected ReadableQueryConfig(ReadableQueryConfig qc, UUID uuid) { - this.queryId = (uuid == null) ? UUID.randomUUID() : uuid; - this.hints = new HashSet<>(); - if (qc == null) { - return; - } - this.distance = qc.distance; - this.distanceWeights = qc.distanceWeights; - this.norm = qc.norm; - this.correspondence = qc.correspondence; - this.resultsPerModule = qc.resultsPerModule; - this.maxResults = qc.maxResults; - this.hints.addAll(qc.hints); - this.relevantSegmentIds.addAll(qc.relevantSegmentIds); - } + private static final int DEFAULT_RESULTS_PER_MODULE = 250; + + /** + * Possible distance functions that can be configured in the {@link QueryConfig}. It's up to the implementing selector to support these distances and / or provide fallback options. + */ + public enum Distance { + chisquared, correlation, cosine, hamming, jaccard, kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, minkowski, spannorm, haversine + } + + /** + * List of Query-Hints that can be configured in the {@link QueryConfig}. It's up to the implementing selector to actually consider these hints. + */ + public enum Hints { + exact, /* Only exact lookup methods should be considered. */ + inexact, /* Inexact lookup methods can be used as well. */ + lsh, ecp, mi, pq, sh, va, vaf, vav, sequential, empirical + } + + private final UUID queryId; + protected Distance distance = null; + protected float[] distanceWeights = null; + protected float norm = Float.NaN; + protected CorrespondenceFunction correspondence = null; + protected int resultsPerModule = -1; + protected Optional maxResults = Optional.empty(); + protected final Set hints; + protected final Set relevantSegmentIds = new HashSet<>(); + + /** + * Constructor used to parse a {@link ReadableQueryConfig} from JSON. + * + * @param queryId The ID of the query. If not set, a new ID will be generated. + * @param hints List of query {@link Hints} used by the query. May be null or empty. + */ + @JsonCreator + public ReadableQueryConfig(@JsonProperty(value = "queryId", required = false) String queryId, + @JsonProperty(value = "hints", required = false) List hints) { + + UUID uuid = null; + try { + uuid = UUID.fromString(queryId); + } catch (IllegalArgumentException | NullPointerException e) { + uuid = UUID.randomUUID(); + } + this.queryId = uuid; + if (hints != null) { + this.hints = new HashSet<>(hints); + } else { + this.hints = new HashSet<>(); + } + } + + /** + * Constructor used to create a {@link ReadableQueryConfig} from another {@link ReadableQueryConfig}. + * + * @param qc The {@link ReadableQueryConfig} that should be used as template + */ + public ReadableQueryConfig(ReadableQueryConfig qc) { + this(qc, qc == null ? null : qc.queryId); + } + + /** + * Internal constructor used to create a {@link ReadableQueryConfig} from another {@link ReadableQueryConfig}. + * + * @param qc The {@link ReadableQueryConfig} that should be used. May be null. + * @param uuid The UUID for the new {@link ReadableQueryConfig}. If null, a new UUID will be created. + */ + protected ReadableQueryConfig(ReadableQueryConfig qc, UUID uuid) { + this.queryId = (uuid == null) ? UUID.randomUUID() : uuid; + this.hints = new HashSet<>(); + if (qc == null) { + return; + } + this.distance = qc.distance; + this.distanceWeights = qc.distanceWeights; + this.norm = qc.norm; + this.correspondence = qc.correspondence; + this.resultsPerModule = qc.resultsPerModule; + this.maxResults = qc.maxResults; + this.hints.addAll(qc.hints); + this.relevantSegmentIds.addAll(qc.relevantSegmentIds); + } + + public final UUID getQueryId() { + return this.queryId; + } + + /** + * Will always be above 0; returns {@link #DEFAULT_RESULTS_PER_MODULE} if no value was set at creation + */ + public int getResultsPerModule() { + return this.resultsPerModule > 0 ? this.resultsPerModule : DEFAULT_RESULTS_PER_MODULE; + } + + /** + * Returns the actual underlying value of {@link #resultsPerModule}, might be below 0 + */ + public int getRawResultsPerModule() { + return this.resultsPerModule; + } + + public Optional getMaxResults() { + return this.maxResults; + } + + public Optional getDistance() { + return Optional.ofNullable(this.distance); + } + + public Optional getNorm() { + return Optional.ofNullable(Float.isNaN(norm) ? null : norm); + } + + + public Optional getCorrespondenceFunction() { + return Optional.ofNullable(this.correspondence); + } + + public Optional getDistanceWeights() { + return Optional.ofNullable(this.distanceWeights); + } + + public Set getHints() { + return this.hints; + } + + /** + * Specifies the segments to which a query result should be limited before any other evaluation. An empty set indicates no additional filtering + * + * @return set of relevant segment ids + */ + public Set getRelevantSegmentIds() { + return Collections.unmodifiableSet(this.relevantSegmentIds); + } + + /** + * checks if the config has segment ids to which the result is to be limited without the need for allocation of immutable wrapper + */ + public boolean hasRelevantSegmentIds() { + return !this.relevantSegmentIds.isEmpty(); + } + + public ReadableQueryConfig withChangesFrom(ReadableQueryConfig other) { - public final UUID getQueryId() { - return this.queryId; + ReadableQueryConfig _return = new ReadableQueryConfig(this); + + if (other.distance != null) { + this.distance = other.distance; } - /** - * Will always be above 0; returns {@link #DEFAULT_RESULTS_PER_MODULE} if no value was set at creation - */ - public int getResultsPerModule() { - return this.resultsPerModule > 0 ? this.resultsPerModule : DEFAULT_RESULTS_PER_MODULE; + if (other.distanceWeights != null) { + this.distanceWeights = other.distanceWeights; + } + + if (!Float.isNaN(other.norm)) { + this.norm = other.norm; } - /** - * Returns the actual underlying value of {@link #resultsPerModule}, might be below 0 - */ - public int getRawResultsPerModule(){ - return this.resultsPerModule; + if (other.resultsPerModule > 0) { + this.resultsPerModule = other.resultsPerModule; } - public Optional getMaxResults() { - return this.maxResults; - } + this.hints.addAll(other.hints); - public Optional getDistance() { - return Optional.ofNullable(this.distance); - } + this.relevantSegmentIds.addAll(other.relevantSegmentIds); - public Optional getNorm() { - return Optional.ofNullable(Float.isNaN(norm) ? null : norm); - } + return _return; - - public Optional getCorrespondenceFunction() { - return Optional.ofNullable(this.correspondence); - } - - public Optional getDistanceWeights() { - return Optional.ofNullable(this.distanceWeights); - } - - public Set getHints() { - return this.hints; - } - - /** - * Specifies the segments to which a query result should be limited before any other evaluation. - * An empty set indicates no additional filtering - * @return set of relevant segment ids - */ - public Set getRelevantSegmentIds() { - return Collections.unmodifiableSet(this.relevantSegmentIds); - } - - /** - * checks if the config has segment ids to which the result is to be limited without the need for allocation of immutable wrapper - */ - public boolean hasRelevantSegmentIds() { - return !this.relevantSegmentIds.isEmpty(); - } - - public ReadableQueryConfig withChangesFrom(ReadableQueryConfig other) { - - ReadableQueryConfig _return = new ReadableQueryConfig(this); - - if (other.distance != null) { - this.distance = other.distance; - } - - if (other.distanceWeights != null) { - this.distanceWeights = other.distanceWeights; - } - - if (!Float.isNaN(other.norm)) { - this.norm = other.norm; - } - - if (other.resultsPerModule > 0) { - this.resultsPerModule = other.resultsPerModule; - } - - this.hints.addAll(other.hints); - - this.relevantSegmentIds.addAll(other.relevantSegmentIds); - - return _return; - - } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/SegmenterConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/SegmenterConfig.java index 6974ecb7a..d783e86ce 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/SegmenterConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/SegmenterConfig.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.Map; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.extraction.ExtractionContextProvider; import org.vitrivr.cineast.core.extraction.segmenter.audio.ConstantLengthAudioSegmenter; @@ -10,85 +12,85 @@ import org.vitrivr.cineast.core.extraction.segmenter.video.VideoHistogramSegmenter; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.HashMap; -import java.util.Map; - public final class SegmenterConfig { - /** Name of the {@link Segmenter} that should be used. Can be null in case there is no specific segmenter class. */ - private final String name; + /** + * Name of the {@link Segmenter} that should be used. Can be null in case there is no specific segmenter class. + */ + private final String name; - /** Properties that should be passed to the {@link Segmenter} upon initialisation. */ - private final Map properties; + /** + * Properties that should be passed to the {@link Segmenter} upon initialisation. + */ + private final Map properties; - /** - * Constructor for {@link SegmenterConfig}. Creates a new default {@link SegmenterConfig} for the - * specified {@link MediaType}. - * - * @param type {@link MediaType} for which to create a {@link SegmenterConfig}. - */ - public SegmenterConfig(MediaType type) { - this.properties = new HashMap<>(); - if(type==null){ - this.name = null; - return; - } - switch (type) { - case AUDIO: - this.name = ConstantLengthAudioSegmenter.class.getName(); - break; - case VIDEO: - this.name = VideoHistogramSegmenter.class.getName(); - break; - case IMAGE: - case MODEL3D: - default: - this.name = null; - } + /** + * Constructor for {@link SegmenterConfig}. Creates a new default {@link SegmenterConfig} for the specified {@link MediaType}. + * + * @param type {@link MediaType} for which to create a {@link SegmenterConfig}. + */ + public SegmenterConfig(MediaType type) { + this.properties = new HashMap<>(); + if (type == null) { + this.name = null; + return; } - - /** - * Constructor for {@link SegmenterConfig}. Used for deserialization of a {@link SegmenterConfig} instance from a configuration file. - * - * @param name The FQN of the {@link Segmenter} class. - * @param properties Properties that should be used to setup the {@link Segmenter} class. - */ - @JsonCreator - public SegmenterConfig(@JsonProperty(value = "name", required = true) String name, - @JsonProperty(value = "properties", required = true) Map properties) { - this.name = name; - this.properties = properties; + switch (type) { + case AUDIO: + this.name = ConstantLengthAudioSegmenter.class.getName(); + break; + case VIDEO: + this.name = VideoHistogramSegmenter.class.getName(); + break; + case IMAGE: + case MODEL3D: + default: + this.name = null; } + } - /** - * Getter for {@link SegmenterConfig#name} - * - * @return {@link SegmenterConfig#name} - */ - public String getName() { - return this.name; - } + /** + * Constructor for {@link SegmenterConfig}. Used for deserialization of a {@link SegmenterConfig} instance from a configuration file. + * + * @param name The FQN of the {@link Segmenter} class. + * @param properties Properties that should be used to setup the {@link Segmenter} class. + */ + @JsonCreator + public SegmenterConfig(@JsonProperty(value = "name", required = true) String name, + @JsonProperty(value = "properties", required = true) Map properties) { + this.name = name; + this.properties = properties; + } - /** - * Getter for {@link SegmenterConfig#properties} - * - * @return {@link SegmenterConfig#properties} - */ - public Map getProperties() { - return this.properties; - } + /** + * Getter for {@link SegmenterConfig#name} + * + * @return {@link SegmenterConfig#name} + */ + public String getName() { + return this.name; + } + + /** + * Getter for {@link SegmenterConfig#properties} + * + * @return {@link SegmenterConfig#properties} + */ + public Map getProperties() { + return this.properties; + } - /** - * Returns a new {@link Segmenter} based on this {@link SegmenterConfig} or null, if the new {@link Segmenter} could not be created. - * - * @return New {@link Segmenter} or null. - */ - @JsonIgnore - public Segmenter newSegmenter(ExtractionContextProvider contextProvider) { - if (this.name != null) { - return ReflectionHelper.newSegmenter(this.name, this.properties, contextProvider); - } else { - return null; - } + /** + * Returns a new {@link Segmenter} based on this {@link SegmenterConfig} or null, if the new {@link Segmenter} could not be created. + * + * @return New {@link Segmenter} or null. + */ + @JsonIgnore + public Segmenter newSegmenter(ExtractionContextProvider contextProvider) { + if (this.name != null) { + return ReflectionHelper.newSegmenter(this.name, this.properties, contextProvider); + } else { + return null; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunction.java index 03d7c002c..4b71ffda4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunction.java @@ -1,10 +1,10 @@ package org.vitrivr.cineast.core.data; import com.google.common.base.Preconditions; - import java.util.function.DoubleUnaryOperator; public class CorrespondenceFunction implements DoubleUnaryOperator { + private static final CorrespondenceFunction IDENTITY = new CorrespondenceFunction(DoubleUnaryOperator.identity()); private final DoubleUnaryOperator function; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DefaultValueHashMap.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DefaultValueHashMap.java index c1b2c2952..1f09f4a5c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DefaultValueHashMap.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DefaultValueHashMap.java @@ -1,10 +1,9 @@ package org.vitrivr.cineast.core.data; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.HashMap; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * HashMap which returns a default value in case the key is unknown @@ -14,9 +13,9 @@ public class DefaultValueHashMap extends HashMap { private static final long serialVersionUID = -8326007409332438019L; private static final Logger LOGGER = LogManager.getLogger(); - + private final V defaultValue; - + public DefaultValueHashMap(V defaultValue) { super(); this.defaultValue = defaultValue; @@ -39,13 +38,12 @@ public DefaultValueHashMap(Map m, V defaultValue) { @Override public V get(Object key) { - if(containsKey(key)){ + if (containsKey(key)) { return super.get(key); } LOGGER.info("key {} not present, returning default value {}", key, defaultValue); return defaultValue; } - - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DoublePair.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DoublePair.java index 86e4f017c..f0680632b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DoublePair.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DoublePair.java @@ -2,16 +2,16 @@ public class DoublePair { - public T key; - public double value; - - public DoublePair(T k, double v){ - this.key = k; - this.value = v; - } - - public static DoublePair pair(V key, double value){ - return new DoublePair(key, value); - } - + public T key; + public double value; + + public DoublePair(T k, double v) { + this.key = k; + this.value = v; + } + + public static DoublePair pair(V key, double value) { + return new DoublePair(key, value); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DynamicGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DynamicGrid.java index 0cf0045e0..6af913000 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DynamicGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/DynamicGrid.java @@ -2,7 +2,6 @@ import gnu.trove.iterator.TIntIterator; import gnu.trove.map.hash.TIntObjectHashMap; - import java.io.Serializable; import java.util.HashSet; import java.util.Set; @@ -20,15 +19,15 @@ public DynamicGrid(T defaultElement) { public DynamicGrid() { this(null); } - - public DynamicGrid(T defaultElement, T[][] data){ + + public DynamicGrid(T defaultElement, T[][] data) { this(defaultElement); - for(int x = 0; x < data.length; ++x){ + for (int x = 0; x < data.length; ++x) { T[] arr = data[x]; TIntObjectHashMap map = new TIntObjectHashMap<>(); - for(int y = 0; y < arr.length; ++y){ + for (int y = 0; y < arr.length; ++y) { T element = arr[y]; - if(element != null){ + if (element != null) { map.put(y, element); } } @@ -75,12 +74,12 @@ public boolean isset(int x, int y) { } return grid.get(x).containsKey(y); } - + /* (non-Javadoc) * @see Grid#remove(int, int) */ @Override - public T remove(int x, int y){ + public T remove(int x, int y) { if (!grid.containsKey(x)) { return null; } @@ -89,24 +88,24 @@ public T remove(int x, int y){ return null; } T _return = map.remove(y); - if(map.isEmpty()){ + if (map.isEmpty()) { grid.remove(x); } return _return; } - + /* (non-Javadoc) * @see Grid#compact() */ @Override - public void compact(){ + public void compact() { TIntIterator iter = grid.keySet().iterator(); - while(iter.hasNext()){ + while (iter.hasNext()) { int x = iter.next(); TIntObjectHashMap map = grid.get(x); - if(map.isEmpty()){ + if (map.isEmpty()) { grid.remove(x); - }else{ + } else { map.compact(); } } @@ -117,10 +116,10 @@ public void compact(){ public Set getKeySet() { HashSet _return = new HashSet<>(); TIntIterator xiter = grid.keySet().iterator(); - while(xiter.hasNext()){ + while (xiter.hasNext()) { int x = xiter.next(); TIntIterator yiter = grid.get(x).keySet().iterator(); - while(yiter.hasNext()){ + while (yiter.hasNext()) { _return.add(new Position(x, yiter.next())); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ExistenceCheck.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ExistenceCheck.java index 2fc3fc5af..5b90ac0cc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ExistenceCheck.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ExistenceCheck.java @@ -2,6 +2,6 @@ public interface ExistenceCheck { - boolean exists(); - + boolean exists(); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FixedSizePriorityQueue.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FixedSizePriorityQueue.java index e730941fb..48ce64915 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FixedSizePriorityQueue.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FixedSizePriorityQueue.java @@ -4,10 +4,7 @@ import java.util.TreeSet; /** - * - * based on - * http://stackoverflow.com/questions/7878026/is-there-a-priorityqueue-implementation-with-fixed- - * capacity-and-custom-comparato + * based on http://stackoverflow.com/questions/7878026/is-there-a-priorityqueue-implementation-with-fixed- capacity-and-custom-comparato */ public class FixedSizePriorityQueue extends TreeSet { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatArrayIterable.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatArrayIterable.java index 83c07b757..9f05368a0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatArrayIterable.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatArrayIterable.java @@ -4,36 +4,36 @@ public class FloatArrayIterable implements Iterable { - private final float[] arr; - - public FloatArrayIterable(float[] arr){ - this.arr = arr; - } - - @Override - public Iterator iterator() { - return new FloatArrayIterator(); - } - - class FloatArrayIterator implements Iterator{ - - int i = 0; - - @Override - public boolean hasNext() { - return i < arr.length; - } - - @Override - public Float next() { - return arr[i++]; - } - - @Override - public void remove() { - //ignore - } - - } + private final float[] arr; + + public FloatArrayIterable(float[] arr) { + this.arr = arr; + } + + @Override + public Iterator iterator() { + return new FloatArrayIterator(); + } + + class FloatArrayIterator implements Iterator { + + int i = 0; + + @Override + public boolean hasNext() { + return i < arr.length; + } + + @Override + public Float next() { + return arr[i++]; + } + + @Override + public void remove() { + //ignore + } + + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVector.java index 4c2e09c69..c1c650d06 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVector.java @@ -1,7 +1,7 @@ package org.vitrivr.cineast.core.data; -public interface FloatVector extends ReadableFloatVector{ - - void setElement(int num, float val); - +public interface FloatVector extends ReadableFloatVector { + + void setElement(int num, float val); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVectorImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVectorImpl.java index 30c61fda2..35b57ff04 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVectorImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/FloatVectorImpl.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.core.data; import gnu.trove.list.array.TFloatArrayList; - import java.util.List; public class FloatVectorImpl implements FloatVector { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java index e438259d1..937cebcf7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/GpsData.java @@ -4,13 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.util.LogHelper; -import org.vitrivr.cineast.core.util.MetadataUtil; -import org.vitrivr.cineast.core.util.OptionalUtil; - -import javax.annotation.Nullable; import java.nio.file.Path; import java.time.Instant; import java.time.format.DateTimeParseException; @@ -18,11 +11,15 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; +import javax.annotation.Nullable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.util.LogHelper; +import org.vitrivr.cineast.core.util.MetadataUtil; +import org.vitrivr.cineast.core.util.OptionalUtil; /** - * Container for GPS data, i.e. location and datetime. The data is extracted from either the Exif - * data, if available, or from a complementary JSON file. See {@link #of(Path)} for more - * information. + * Container for GPS data, i.e. location and datetime. The data is extracted from either the Exif data, if available, or from a complementary JSON file. See {@link #of(Path)} for more information. */ public class GpsData { @@ -48,13 +45,10 @@ private GpsData(@Nullable Location location, @Nullable Instant time) { } /** - * Extracts the GPS data from a given file using Exif. If the Exif data is incomplete, additional - * data is retrieved from the complementary JSON file named after the original file, e.g. {@code - * image_0001.json} for {@code image_0001.jpg}. + * Extracts the GPS data from a given file using Exif. If the Exif data is incomplete, additional data is retrieved from the complementary JSON file named after the original file, e.g. {@code image_0001.json} for {@code image_0001.jpg}. * * @param file file to extract data from - * @return an object containing the extracted information, if available, otherwise an empty - * object. + * @return an object containing the extracted information, if available, otherwise an empty object. * @see #ofExif(Path) * @see #ofJson(Path) */ @@ -63,12 +57,10 @@ public static GpsData of(Path file) { } /** - * Extracts the GPS data from the given file by reading the available Exif data. In particular, - * latitude, longitude, date and time stamp are read from the file. + * Extracts the GPS data from the given file by reading the available Exif data. In particular, latitude, longitude, date and time stamp are read from the file. * * @param file file to extract exif data from - * @return an object containing the extracted information, if available, otherwise an empty - * object. + * @return an object containing the extracted information, if available, otherwise an empty object. */ public static GpsData ofExif(Path file) { GpsDirectory gps = MetadataUtil.getMetadataDirectoryOfType(file, GpsDirectory.class); @@ -81,9 +73,7 @@ public static GpsData ofExif(Path file) { } /** - * Extracts the GPS data from the complementary JSON file named after the original file, e.g. - * {@code image_0001.json} for {@code image_0001.jpg}. The method expects a top-level JSON object - * in the form of + * Extracts the GPS data from the complementary JSON file named after the original file, e.g. {@code image_0001.json} for {@code image_0001.jpg}. The method expects a top-level JSON object in the form of * * 

        * {
@@ -94,30 +84,26 @@ public static GpsData ofExif(Path file) {
    * }
    * *

    Location information requires the key {@code "location"} containing an object with numbers - * for {@code "latitude"} and {@code "longitude"}. Alternatively, the coordinates can also be - * represented as an two-valued array with latitude first, longitude second, such as {@code - * [47.559601, 7.588576]}. - * - * Time information must either be a valid ISO 8601 instant description, with key "{@code datetime}" or - * conform to the description of {@link #parseFuzzyDating(String)}. + * for {@code "latitude"} and {@code "longitude"}. Alternatively, the coordinates can also be represented as an two-valued array with latitude first, longitude second, such as {@code [47.559601, 7.588576]}. + *

    + * Time information must either be a valid ISO 8601 instant description, with key "{@code datetime}" or conform to the description of {@link #parseFuzzyDating(String)}. * * @param file file to extract json data from - * @return an object containing the extracted information, if available, otherwise an empty - * object. + * @return an object containing the extracted information, if available, otherwise an empty object. */ public static GpsData ofJson(Path file) { Optional root = MetadataUtil.getJsonMetadata(file); - + Optional instant; - - if(root.isPresent() && root.get().has(KEY_DATETIME)){ + + if (root.isPresent() && root.get().has(KEY_DATETIME)) { instant = root .map(o -> o.get(KEY_DATETIME)) .map(JsonNode::textValue) .flatMap(GpsData::parseInstant); - }else if(root.isPresent() && root.get().hasNonNull(KEY_DATING)){ - instant = parseFuzzyDating(root.get().get(KEY_DATING).asText() ); - }else{ + } else if (root.isPresent() && root.get().hasNonNull(KEY_DATING)) { + instant = parseFuzzyDating(root.get().get(KEY_DATING).asText()); + } else { instant = Optional.empty(); } @@ -127,7 +113,7 @@ public static GpsData ofJson(Path file) { return ofData(location.orElse(null), instant.orElse(null)); } - + /** * Parses fuzzy Dating specifiers. Several input formats are accepted: * @@ -138,7 +124,7 @@ public static GpsData ofJson(Path file) { *

  • English annual estimate: {@code (before | approx. | after) YYYY}
    • * * The annual estimates are respected with best effort. - * + *

    * If the Dating given does not contain information for *

      *
    • the day of the month, the 15th is then used
      • @@ -148,75 +134,75 @@ public static GpsData ofJson(Path file) { *
    * Following this approach, the date is somewhat 'centered' within a year, so that the {@link org.vitrivr.cineast.core.features.TemporalDistance} works best. */ - public static Optional parseFuzzyDating(String dating){ - if(dating == null){ + public static Optional parseFuzzyDating(String dating) { + if (dating == null) { return Optional.empty(); - }else if(dating.isEmpty()){ + } else if (dating.isEmpty()) { return Optional.empty(); } - logger.debug("Parsing dating: "+dating); - String[] prePrefixes = new String[]{"pre","vor","before"}; + logger.debug("Parsing dating: " + dating); + String[] prePrefixes = new String[]{"pre", "vor", "before"}; String[] approxPrefixes = new String[]{"ca.", "ca", "circa", "approx.", "approx", "approximately", "about", "around"}; String[] postPrefixes = new String[]{"post", "nach", "after"}; - - int year=-1, month=6, day=15, hours=12, minutes=0, seconds=0; - if(dating.contains("/")){ + + int year = -1, month = 6, day = 15, hours = 12, minutes = 0, seconds = 0; + if (dating.contains("/")) { /* * US-style M/YYYY dating specification */ String[] splits = dating.split("/"); - if(splits.length >= 2){ + if (splits.length >= 2) { try { month = Integer.parseInt(splits[0]); - if(month < 1 || month > 12){ - throw new NumberFormatException("Month out of bounds: "+month); + if (month < 1 || month > 12) { + throw new NumberFormatException("Month out of bounds: " + month); } - }catch(NumberFormatException e){ - logger.error("Couldn't parse month value "+splits[0]); + } catch (NumberFormatException e) { + logger.error("Couldn't parse month value " + splits[0]); logger.catching(e); month = 6; } - try{ + try { year = Integer.parseInt(splits[1]); - if(year < 0){ - throw new NumberFormatException("Year cannot be negative "+year); + if (year < 0) { + throw new NumberFormatException("Year cannot be negative " + year); } - }catch(NumberFormatException e){ - logger.error("Couldn't parse year value: "+splits[1]); + } catch (NumberFormatException e) { + logger.error("Couldn't parse year value: " + splits[1]); logger.catching(e); year = -1; } } - }else{ + } else { /* * Annual estimate, as described above */ String[] splits = dating.split(" "); - if(splits.length >= 2){ + if (splits.length >= 2) { int modifier = 0; - if(Arrays.asList(prePrefixes).contains(splits[0])){ + if (Arrays.asList(prePrefixes).contains(splits[0])) { modifier = -1; - }else if(Arrays.asList(approxPrefixes).contains(splits[0])){ + } else if (Arrays.asList(approxPrefixes).contains(splits[0])) { modifier = 0; - }else if(Arrays.asList(postPrefixes).contains(splits[0])){ + } else if (Arrays.asList(postPrefixes).contains(splits[0])) { modifier = 1; - }else{ - logger.error("Unknown Dating prefix: "+splits[0]); + } else { + logger.error("Unknown Dating prefix: " + splits[0]); } - try{ + try { int _year = Integer.parseInt(splits[1].trim()); year = modifier + _year; - }catch(NumberFormatException e){ - logger.error("Year not parsable: "+splits[1]); + } catch (NumberFormatException e) { + logger.error("Year not parsable: " + splits[1]); logger.catching(e); } } } - if(year!=-1){ - String instant = String.format("%d-%02d-%02dT%02d:%02d:%02d.00Z",year,month,day,hours,minutes,seconds); + if (year != -1) { + String instant = String.format("%d-%02d-%02dT%02d:%02d:%02d.00Z", year, month, day, hours, minutes, seconds); logger.debug("Fuzzy instant pre-parsing: {}", instant); return parseInstant(instant); - }else{ + } else { return Optional.empty(); } } @@ -227,8 +213,7 @@ public static Optional parseFuzzyDating(String dating){ *

    The string must represent a valid instant in UTC and is parsed using {@link Instant#parse}. * * @param string string to parse - * @return an {@link Optional} containing a {@code Instant}, if the given text contains a valid - * instant, otherwise an empty {@code Optional}. + * @return an {@link Optional} containing a {@code Instant}, if the given text contains a valid instant, otherwise an empty {@code Optional}. */ public static Optional parseInstant(String string) { try { @@ -240,18 +225,14 @@ public static Optional parseInstant(String string) { } /** - * Extracts location information of the given {@link JsonNode} from the {@code latitude} and - * {@code longitude} fields. If not available or valid, the coordinates are extracted as an - * two-valued array with latitude first, longitude second. For example, the JsonNode may look like - * the following: + * Extracts location information of the given {@link JsonNode} from the {@code latitude} and {@code longitude} fields. If not available or valid, the coordinates are extracted as an two-valued array with latitude first, longitude second. For example, the JsonNode may look like the following: * * 

    {"latitude": 42.43, "longitude": 7.6}
    * or * 
    [42.43, 7.6]
    * * @param node json node to extract location information from - * @return an {@link Optional} with a {@link Location}, if both "latitude" and "longitude" - * coordinates are available and valid, otherwise an empty {@code Optional}. + * @return an {@link Optional} with a {@link Location}, if both "latitude" and "longitude" coordinates are available and valid, otherwise an empty {@code Optional}. */ public static Optional parseLocationFromJson(JsonNode node) { return OptionalUtil.or( @@ -298,12 +279,10 @@ public Optional time() { } /** - * Returns this if all values are present, otherwise invoke {@code other} and set all undefined - * values from the result of that invocation. + * Returns this if all values are present, otherwise invoke {@code other} and set all undefined values from the result of that invocation. * * @param other a {@code Supplier} whose result is used to set undefined values - * @return this if all values are present, otherwise a {@code GpsData} whose data is set by this - * or alternatively by the results of {@code other.get()} + * @return this if all values are present, otherwise a {@code GpsData} whose data is set by this or alternatively by the results of {@code other.get()} * @throws NullPointerException if {@code other} or the result of {@code other} is null */ public GpsData orElse(Supplier other) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java index e5780fc80..4cefa6ffc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java @@ -12,7 +12,8 @@ public interface Grid { /** * returns and removes the element at (x,y). - * @return the element at position (x,y) or null if there was no such element + * + * @return the element at position (x,y) or null if there was no such element */ T remove(int x, int y); @@ -20,7 +21,7 @@ public interface Grid { * compacts the grid by removing unnecessary data structures. */ void compact(); - + Set getKeySet(); } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Histogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Histogram.java index d8e8a491c..c23ad5e3b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Histogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Histogram.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.core.data; import gnu.trove.map.hash.TObjectIntHashMap; - import java.util.NoSuchElementException; public abstract class Histogram implements ReadableFloatVector { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/InstantVector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/InstantVector.java index aef96b977..c1f05e992 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/InstantVector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/InstantVector.java @@ -3,6 +3,7 @@ import java.time.Instant; public class InstantVector implements ReadableFloatVector { + private static final int ELEMENT_COUNT = 1; private final Instant instant; @@ -27,8 +28,10 @@ public int getElementCount() { @Override public float getElement(int num) { switch (num) { - case 0: return instant.getEpochSecond(); - default: throw new IndexOutOfBoundsException(num + " >= " + this.getElementCount()); + case 0: + return instant.getEpochSecond(); + default: + throw new IndexOutOfBoundsException(num + " >= " + this.getElementCount()); } } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/IntArrayIterable.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/IntArrayIterable.java index 139183a6e..e9494ce0f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/IntArrayIterable.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/IntArrayIterable.java @@ -4,36 +4,36 @@ public class IntArrayIterable implements Iterable { - private final int[] arr; - - public IntArrayIterable(int[] arr){ - this.arr = arr; - } - - @Override - public Iterator iterator() { - return new IntArrayIterator(); - } - - class IntArrayIterator implements Iterator{ - - int i = 0; - - @Override - public boolean hasNext() { - return i < arr.length; - } - - @Override - public Integer next() { - return arr[i++]; - } - - @Override - public void remove() { - //ignore - } - - } + private final int[] arr; + + public IntArrayIterable(int[] arr) { + this.arr = arr; + } + + @Override + public Iterator iterator() { + return new IntArrayIterator(); + } + + class IntArrayIterator implements Iterator { + + int i = 0; + + @Override + public boolean hasNext() { + return i < arr.length; + } + + @Override + public Integer next() { + return arr[i++]; + } + + @Override + public void remove() { + //ignore + } + + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/LimitedQueue.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/LimitedQueue.java index 79495a2e2..f4156dae4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/LimitedQueue.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/LimitedQueue.java @@ -3,34 +3,31 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -public class LimitedQueue extends LinkedBlockingQueue implements BlockingQueue -{ - /** - * http://stackoverflow.com/questions/4521983/java-executorservice-that-blocks-on-submission-after-a-certain-queue-size - */ - private static final long serialVersionUID = -107554551692604207L; +public class LimitedQueue extends LinkedBlockingQueue implements BlockingQueue { - public LimitedQueue(int maxSize) - { - super(maxSize); - } + /** + * http://stackoverflow.com/questions/4521983/java-executorservice-that-blocks-on-submission-after-a-certain-queue-size + */ + private static final long serialVersionUID = -107554551692604207L; + + public LimitedQueue(int maxSize) { + super(maxSize); + } - /** - * blocking - */ - @Override - public boolean offer(E e) - { - // turn offer() and add() into a blocking calls (unless interrupted) - try { - put(e); - return true; - } catch(InterruptedException ie) { - Thread.currentThread().interrupt(); - } - return false; + /** + * blocking + */ + @Override + public boolean offer(E e) { + // turn offer() and add() into a blocking calls (unless interrupted) + try { + put(e); + return true; + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); } - - + return false; + } + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Location.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Location.java index 8681dca44..b96120d96 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Location.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Location.java @@ -2,13 +2,13 @@ import com.drew.lang.GeoLocation; import com.google.common.base.Preconditions; +import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.Objects; - public class Location implements ReadableFloatVector { + private static final int ELEMENT_COUNT = 2; private static final Logger logger = LogManager.getLogger(); @@ -82,9 +82,12 @@ public int getElementCount() { @Override public float getElement(int num) { switch (num) { - case 0: return this.getLatitude(); - case 1: return this.getLongitude(); - default: throw new IndexOutOfBoundsException(num + " >= " + this.getElementCount()); + case 0: + return this.getLatitude(); + case 1: + return this.getLongitude(); + default: + throw new IndexOutOfBoundsException(num + " >= " + this.getElementCount()); } } @@ -97,7 +100,7 @@ public boolean equals(Object o) { return false; } Location location = (Location) o; - return Float.compare(location.latitude, latitude) == 0 + return Float.compare(location.latitude, latitude) == 0 && Float.compare(location.longitude, longitude) == 0; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java index d538e72ca..3cefb4cba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java @@ -1,204 +1,200 @@ package org.vitrivr.cineast.core.data; import gnu.trove.map.hash.TIntObjectHashMap; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; - import java.util.EnumSet; import java.util.HashMap; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; public enum MediaType { - VIDEO(0, "v", "video"), - IMAGE(1, "i", "image"), - AUDIO(2, "a", "audio"), - MODEL3D(3, "m", "3dmodel"), - IMAGE_SEQUENCE(4, "is", "imagesequence"), - UNKNOWN(99, "u", "unknown"); - - private final int id; - private final String prefix; - private final String name; - - /** - * Constructs a new media type given the id, the prefix and the name. - * - * @param id numeric id to use to identify the type in the persistent storage layer - * @param prefix the prefix of the MediaType without a trailing delimiter such as '_' - * @param name the name of the media type - */ - MediaType(int id, String prefix, String name) { - this.id = id; - this.prefix = prefix; - this.name = name.trim(); - } - - public int getId() { - return this.id; - } - - /** - * @return the prefix of the MediaType without a trailing delimiter such as '_'. - */ - public String getPrefix() { - return this.prefix; - } - - public String getName() { - return this.name; - } - - private static final TIntObjectHashMap idToType = new TIntObjectHashMap<>(); - private static final HashMap prefixToType = new HashMap<>(); - private static final HashMap nameToType = new HashMap<>(); - - public static final char DELIMITER = '_'; - - static { - for (MediaType t : EnumSet.allOf(MediaType.class)) { - if (idToType.containsKey(t.getId())) { - throw new IllegalStateException("duplicate id (" + t.getId() + ") in Mediatype: " + t - + " collides with " + idToType.get(t.getId())); - } - idToType.put(t.getId(), t); - - if (prefixToType.containsKey(t.getPrefix())) { - throw new IllegalStateException("duplicate prefix (" + t.getPrefix() + ") in Mediatype: " - + t + " collides with " + prefixToType.get(t.getPrefix())); - } - prefixToType.put(t.getPrefix(), t); - - if (nameToType.containsKey(t.getPrefix())) { - throw new IllegalStateException("duplicate name (" + t.getName() + ") in Mediatype: " + t - + " collides with " + nameToType.get(t.getName())); - } - nameToType.put(t.getName().toLowerCase(), t); - } - } - - /** - * @return the MediaType associated with this id or null in case there is none. - */ - public static MediaType fromId(int id) { - return idToType.get(id); - } - - /** - * @return the MediaType associated with this prefix or null in case there is none. - */ - public static MediaType fromPrefix(String prefix) { - if (prefix == null) { - return null; - } - return prefixToType.get(prefix.trim().toLowerCase()); - } - - /** - * @return the MediaType associated with this name or null in case there is none. - */ - public static MediaType fromName(String name) { - if (name == null) { - return null; - } - return nameToType.get(name.trim().toLowerCase()); - } - - public static boolean existsId(int id) { - return idToType.containsKey(id); - } - - public static boolean existsPrefix(String prefix) { - return prefixToType.containsKey(prefix); - } - - public static boolean existsName(String name) { - return nameToType.containsKey(name.trim().toLowerCase()); - } - - /** - * generates an id of the form (prefix)_(object id) assuming the delimiter is '_'. - * - * @param type the type for which an id is to be generated - * @param objectId the globally unique id of the object - * @throws IllegalArgumentException if objectId is empty - * @throws NullPointerException if type or objectId is null - */ - public static String generateId(MediaType type, String objectId) throws IllegalArgumentException, NullPointerException { - if (type == null) { - throw new NullPointerException("Media type cannot be null!"); - } - if (objectId == null) { - throw new NullPointerException("Object ID cannot be null!"); - } - if (objectId.isEmpty()) { - throw new IllegalArgumentException("Object ID must not be empty!"); - } - - return type.getPrefix() + DELIMITER + objectId; - } - - /** - * Parses the provided object ID and decomposes it into a pair containing the {@link MediaType}, the ID of the media object. - * - * @param objectId The object ID that should be parsed. - * @return Resulting pair. - */ - public static Pair parseObjectId(String objectId) { - final String[] components = objectId.split(String.valueOf(DELIMITER)); - return new ImmutablePair<>(MediaType.fromPrefix(components[0]), components[1]); - } - - /** - * generates a segment id of the form (prefix)_(object id)_(sequence number) assuming the - * delimiter is '_'. - * - * @param type the type for which an id is to be generated - * @param objectId the globally unique id of the object (without MediaType prefix) - * @param sequenceNumber the number of the segment within the object - * @throws IllegalArgumentException if shot sequence number is negative or objectId is empty - * @throws NullPointerException if type or objectId is null - */ - public static String generateSegmentId(MediaType type, String objectId, long sequenceNumber) - throws IllegalArgumentException, NullPointerException { - if (sequenceNumber < 0) { - throw new IllegalArgumentException("sequenceNumber must be non-negative"); - } - - return generateId(type, objectId) + DELIMITER + sequenceNumber; - - } - - /** - * generates a segment id of the form (prefix)_(object id)_(sequence number) assuming the - * delimiter is '_' - * - * @param objectId the globally unique id of the object (with MediaType prefix) - * @param sequenceNumber the number of the segment within the object - * @throws IllegalArgumentException if shot sequence number is negative or objectId is empty - * @throws NullPointerException if type or objectId is null - */ - public static String generateSegmentId(String objectId, long sequenceNumber) throws IllegalArgumentException, NullPointerException { - if (sequenceNumber < 0) { - throw new IllegalArgumentException("sequenceNumber must be non-negative"); - } - return objectId + DELIMITER + sequenceNumber; - } - - /** - * Parses the provided segmentId and decomposes it into a pair containing the ID of the media object - * and the segment's sequence number. - * - * @param segmentId The segment ID that should be parsed. - * @return Resulting pair. - */ - public static Pair parsesSegmentId(String segmentId) { - final String[] components = segmentId.split(String.valueOf(DELIMITER)); - StringBuilder sb = new StringBuilder(); - sb.append(components[0]); - for(int i = 1; i < components.length - 1; ++i){ - sb.append(DELIMITER); - sb.append(components[i]); - } - return new ImmutablePair<>(sb.toString(), Long.valueOf(components[components.length - 1])); - } + VIDEO(0, "v", "video"), + IMAGE(1, "i", "image"), + AUDIO(2, "a", "audio"), + MODEL3D(3, "m", "3dmodel"), + IMAGE_SEQUENCE(4, "is", "imagesequence"), + UNKNOWN(99, "u", "unknown"); + + private final int id; + private final String prefix; + private final String name; + + /** + * Constructs a new media type given the id, the prefix and the name. + * + * @param id numeric id to use to identify the type in the persistent storage layer + * @param prefix the prefix of the MediaType without a trailing delimiter such as '_' + * @param name the name of the media type + */ + MediaType(int id, String prefix, String name) { + this.id = id; + this.prefix = prefix; + this.name = name.trim(); + } + + public int getId() { + return this.id; + } + + /** + * @return the prefix of the MediaType without a trailing delimiter such as '_'. + */ + public String getPrefix() { + return this.prefix; + } + + public String getName() { + return this.name; + } + + private static final TIntObjectHashMap idToType = new TIntObjectHashMap<>(); + private static final HashMap prefixToType = new HashMap<>(); + private static final HashMap nameToType = new HashMap<>(); + + public static final char DELIMITER = '_'; + + static { + for (MediaType t : EnumSet.allOf(MediaType.class)) { + if (idToType.containsKey(t.getId())) { + throw new IllegalStateException("duplicate id (" + t.getId() + ") in Mediatype: " + t + + " collides with " + idToType.get(t.getId())); + } + idToType.put(t.getId(), t); + + if (prefixToType.containsKey(t.getPrefix())) { + throw new IllegalStateException("duplicate prefix (" + t.getPrefix() + ") in Mediatype: " + + t + " collides with " + prefixToType.get(t.getPrefix())); + } + prefixToType.put(t.getPrefix(), t); + + if (nameToType.containsKey(t.getPrefix())) { + throw new IllegalStateException("duplicate name (" + t.getName() + ") in Mediatype: " + t + + " collides with " + nameToType.get(t.getName())); + } + nameToType.put(t.getName().toLowerCase(), t); + } + } + + /** + * @return the MediaType associated with this id or null in case there is none. + */ + public static MediaType fromId(int id) { + return idToType.get(id); + } + + /** + * @return the MediaType associated with this prefix or null in case there is none. + */ + public static MediaType fromPrefix(String prefix) { + if (prefix == null) { + return null; + } + return prefixToType.get(prefix.trim().toLowerCase()); + } + + /** + * @return the MediaType associated with this name or null in case there is none. + */ + public static MediaType fromName(String name) { + if (name == null) { + return null; + } + return nameToType.get(name.trim().toLowerCase()); + } + + public static boolean existsId(int id) { + return idToType.containsKey(id); + } + + public static boolean existsPrefix(String prefix) { + return prefixToType.containsKey(prefix); + } + + public static boolean existsName(String name) { + return nameToType.containsKey(name.trim().toLowerCase()); + } + + /** + * generates an id of the form (prefix)_(object id) assuming the delimiter is '_'. + * + * @param type the type for which an id is to be generated + * @param objectId the globally unique id of the object + * @throws IllegalArgumentException if objectId is empty + * @throws NullPointerException if type or objectId is null + */ + public static String generateId(MediaType type, String objectId) throws IllegalArgumentException, NullPointerException { + if (type == null) { + throw new NullPointerException("Media type cannot be null!"); + } + if (objectId == null) { + throw new NullPointerException("Object ID cannot be null!"); + } + if (objectId.isEmpty()) { + throw new IllegalArgumentException("Object ID must not be empty!"); + } + + return type.getPrefix() + DELIMITER + objectId; + } + + /** + * Parses the provided object ID and decomposes it into a pair containing the {@link MediaType}, the ID of the media object. + * + * @param objectId The object ID that should be parsed. + * @return Resulting pair. + */ + public static Pair parseObjectId(String objectId) { + final String[] components = objectId.split(String.valueOf(DELIMITER)); + return new ImmutablePair<>(MediaType.fromPrefix(components[0]), components[1]); + } + + /** + * generates a segment id of the form (prefix)_(object id)_(sequence number) assuming the delimiter is '_'. + * + * @param type the type for which an id is to be generated + * @param objectId the globally unique id of the object (without MediaType prefix) + * @param sequenceNumber the number of the segment within the object + * @throws IllegalArgumentException if shot sequence number is negative or objectId is empty + * @throws NullPointerException if type or objectId is null + */ + public static String generateSegmentId(MediaType type, String objectId, long sequenceNumber) + throws IllegalArgumentException, NullPointerException { + if (sequenceNumber < 0) { + throw new IllegalArgumentException("sequenceNumber must be non-negative"); + } + + return generateId(type, objectId) + DELIMITER + sequenceNumber; + + } + + /** + * generates a segment id of the form (prefix)_(object id)_(sequence number) assuming the delimiter is '_' + * + * @param objectId the globally unique id of the object (with MediaType prefix) + * @param sequenceNumber the number of the segment within the object + * @throws IllegalArgumentException if shot sequence number is negative or objectId is empty + * @throws NullPointerException if type or objectId is null + */ + public static String generateSegmentId(String objectId, long sequenceNumber) throws IllegalArgumentException, NullPointerException { + if (sequenceNumber < 0) { + throw new IllegalArgumentException("sequenceNumber must be non-negative"); + } + return objectId + DELIMITER + sequenceNumber; + } + + /** + * Parses the provided segmentId and decomposes it into a pair containing the ID of the media object and the segment's sequence number. + * + * @param segmentId The segment ID that should be parsed. + * @return Resulting pair. + */ + public static Pair parsesSegmentId(String segmentId) { + final String[] components = segmentId.split(String.valueOf(DELIMITER)); + StringBuilder sb = new StringBuilder(); + sb.append(components[0]); + for (int i = 1; i < components.length - 1; ++i) { + sb.append(DELIMITER); + sb.append(components[i]); + } + return new ImmutablePair<>(sb.toString(), Long.valueOf(components[components.length - 1])); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Pair.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Pair.java index dec1f69d5..6ff0aca18 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Pair.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Pair.java @@ -2,12 +2,12 @@ public class Pair { - public K first; - public V second; - - public Pair(K first, V second){ - this.first = first; - this.second = second; - } - + public K first; + public V second; + + public Pair(K first, V second) { + this.first = first; + this.second = second; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java index 532abdf1c..e91355589 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java @@ -41,7 +41,7 @@ public Position getPosTop() { } public Position[] getNeighborPositions() { - return new Position[] { getPosTop(), getPosLeft(), getPosBottom(), getPosRight() }; + return new Position[]{getPosTop(), getPosLeft(), getPosBottom(), getPosRight()}; } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/QuerySubTitleItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/QuerySubTitleItem.java index bcdebe865..5ddf63b49 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/QuerySubTitleItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/QuerySubTitleItem.java @@ -4,29 +4,29 @@ public class QuerySubTitleItem implements SubtitleItem { - private final String text; - - public QuerySubTitleItem(String text){ - this.text = text; - } - - @Override - public int getLength() { - return 0; - } - - @Override - public String getText() { - return text; - } - - @Override - public long getStartTimestamp() { - return 0; - } - - @Override - public long getEndTimestamp() { - return 0; - } + private final String text; + + public QuerySubTitleItem(String text) { + this.text = text; + } + + @Override + public int getLength() { + return 0; + } + + @Override + public String getText() { + return text; + } + + @Override + public long getStartTimestamp() { + return 0; + } + + @Override + public long getEndTimestamp() { + return 0; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ReadableFloatVector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ReadableFloatVector.java index ebe6c5f24..569dfd3a6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ReadableFloatVector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/ReadableFloatVector.java @@ -19,8 +19,7 @@ static float[] toArray(ReadableFloatVector vector) { /** * @param vector the vector - * @param array the array to write into. If {@code array} does not match the vector size, a new - * array is generated instead. + * @param array the array to write into. If {@code array} does not match the vector size, a new array is generated instead. * @return an array containing the vector elements */ static float[] toArray(ReadableFloatVector vector, float[] array) { @@ -44,7 +43,7 @@ static List toList(ReadableFloatVector vector) { /** * @param vector the vector - * @param list the list to write into. Note that existing elements are removed from {@code list}. + * @param list the list to write into. Note that existing elements are removed from {@code list}. * @return the list containing the vector elements */ static List toList(ReadableFloatVector vector, List list) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/SemanticMap.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/SemanticMap.java index c958fafb7..4597ebe40 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/SemanticMap.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/SemanticMap.java @@ -1,11 +1,10 @@ package org.vitrivr.cineast.core.data; import gnu.trove.map.hash.TIntObjectHashMap; -import org.vitrivr.cineast.core.color.ReadableRGBContainer; -import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel; - import java.awt.image.BufferedImage; import java.util.Map; +import org.vitrivr.cineast.core.color.ReadableRGBContainer; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel; public class SemanticMap { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/StringDoublePair.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/StringDoublePair.java index bf9910d56..5ff2e4570 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/StringDoublePair.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/StringDoublePair.java @@ -3,23 +3,23 @@ import java.util.Comparator; public class StringDoublePair extends DoublePair { - - @Override - public String toString() { - return "StringDoublePair(" + key + ", " + value + ")"; - } - - public StringDoublePair(String k, double v){ - super(k, v); - } - - public static final Comparator COMPARATOR = new Comparator(){ - - @Override - public int compare(StringDoublePair o1, StringDoublePair o2) { - return Double.compare(o2.value, o1.value); - } - - }; - + + @Override + public String toString() { + return "StringDoublePair(" + key + ", " + value + ")"; + } + + public StringDoublePair(String k, double v) { + super(k, v); + } + + public static final Comparator COMPARATOR = new Comparator() { + + @Override + public int compare(StringDoublePair o1, StringDoublePair o2) { + return Double.compare(o2.value, o1.value); + } + + }; + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java index 1256cdf3c..2252aed03 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.core.data; import com.google.common.collect.HashBiMap; - import java.util.HashSet; import java.util.Set; @@ -14,7 +13,7 @@ public Grid setElement(int x, int y, T element) { map.forcePut(new Position(x, y), element); return this; } - + public Grid setElement(Position p, T element) { map.forcePut(p, element); return this; @@ -24,9 +23,9 @@ public Grid setElement(Position p, T element) { public T get(int x, int y) { return map.get(new Position(x, y)); } - - public T get(Position p){ - if(p == null){ + + public T get(Position p) { + if (p == null) { return null; } return map.get(p); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/AbstractDistanceElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/AbstractDistanceElement.java index e46383871..f871ae70e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/AbstractDistanceElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/AbstractDistanceElement.java @@ -3,6 +3,7 @@ import java.util.Objects; abstract class AbstractDistanceElement implements DistanceElement { + private final String id; private final double distance; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/DistanceElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/DistanceElement.java index e913907fe..314b08d0c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/DistanceElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/DistanceElement.java @@ -1,16 +1,16 @@ package org.vitrivr.cineast.core.data.distance; -import org.apache.logging.log4j.LogManager; -import org.vitrivr.cineast.core.data.CorrespondenceFunction; -import org.vitrivr.cineast.core.data.score.ScoreElement; -import org.vitrivr.cineast.core.util.GroupingUtil; - import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.logging.log4j.LogManager; +import org.vitrivr.cineast.core.data.CorrespondenceFunction; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.util.GroupingUtil; public interface DistanceElement { + String getId(); double getDistance(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/ObjectDistanceElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/ObjectDistanceElement.java index 9e186e46b..4505e575b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/ObjectDistanceElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/distance/ObjectDistanceElement.java @@ -4,6 +4,7 @@ import org.vitrivr.cineast.core.data.score.ObjectScoreElement; public class ObjectDistanceElement extends AbstractDistanceElement { + public ObjectDistanceElement(String id, double distance) { super(id, distance); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java index d571568a5..2ff4f1d34 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java @@ -11,7 +11,6 @@ import org.vitrivr.cineast.core.data.ExistenceCheck; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; -import org.vitrivr.cineast.core.db.setup.EntityDefinition; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; @@ -38,9 +37,9 @@ public class MediaObjectDescriptor implements ExistenceCheck { * Convenience method to create a MediaObjectDescriptor marked as new. The method will assign a new ID to this MediaObjectDescriptor using the provided ObjectIdGenerator. * * @param generator ObjectIdGenerator used for ID generation. - * @param path The Path that points to the file for which a new MediaObjectDescriptor should be created. - * @param type MediaType of the new MediaObjectDescriptor - * @param lookup MediaObjectReader to prevent the assignment of already used ids + * @param path The Path that points to the file for which a new MediaObjectDescriptor should be created. + * @param type MediaType of the new MediaObjectDescriptor + * @param lookup MediaObjectReader to prevent the assignment of already used ids * @return A new MediaObjectDescriptor */ public static MediaObjectDescriptor newMultimediaObjectDescriptor( diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaSegmentDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaSegmentDescriptor.java index 5be7a79d1..6f1a3f29f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaSegmentDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaSegmentDescriptor.java @@ -10,128 +10,133 @@ import org.vitrivr.cineast.core.data.ExistenceCheck; /** - * Describes a media segment in the Cineast data model i.e. a part of a media object (e.g. a series of frames in a video - * or an image in an image sequence). The cardinality of a media object to its segments is 1 to (1..n) depending on the - * type of media. - * + * Describes a media segment in the Cineast data model i.e. a part of a media object (e.g. a series of frames in a video or an image in an image sequence). The cardinality of a media object to its segments is 1 to (1..n) depending on the type of media. */ public final class MediaSegmentDescriptor implements ExistenceCheck { - /** Name of the entity in the persistence layer. */ - public static final String ENTITY = "cineast_segment"; - - /** Name of the column with the segment number */ - public static final String SEGMENT_NO_COL_NAME = "segmentnumber"; - /** Name of the column with segmentstart value*/ - public static final String SEGMENT_START_COL_NAME = "segmentstart"; - /** Name of the column with segmentend value */ - public static final String SEGMENT_END_COL_NAME = "segmentend"; - /** Name of the column with segmetnstartabs value */ - public static final String SEGMENT_STARTABS_COL_NAME = "segmentstartabs"; - /** Name of the column with segmentendabs value*/ - public static final String SEGMENT_ENDABS_COL_NAME = "segmentendabs"; - - /** Field names in the persistence layer. - * - * Order (Important): - * - segmentid (PK) - * - objectid (FK -> MediaObject) - * - segmentnumber - * - segmentstart - * - segmentend - * - startabs - */ - public static final String[] FIELDNAMES = { - SEGMENT_ID_COLUMN_QUALIFIER, - OBJECT_ID_COLUMN_QUALIFIER, - SEGMENT_NO_COL_NAME, - SEGMENT_START_COL_NAME, - SEGMENT_END_COL_NAME, - SEGMENT_STARTABS_COL_NAME, - SEGMENT_ENDABS_COL_NAME}; - - private final String segmentId, objectId; - private final int start, end, number; - private final float startabs, endabs; - private final boolean exists; - - /** - * Constructor for {@link MediaSegmentDescriptor}. - * - * @param objectId ID of the {@link MediaObjectDescriptor} this {@link MediaSegmentDescriptor} belongs to. - * @param segmentId ID of the {@link MediaSegmentDescriptor}. - * @param segmentNumber Relative position of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} (starts with 1) - * @param start Start of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in frames (e.g. for videos or audio). - * @param end End of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in frames (e.g. for videos or audio). - * @param startabs Absolute start of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in seconds (e.g. for videos or audio). - * @param endabs Absolute end of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in seconds (e.g. for videos or audio). - * @param exists Whether or not this {@link MediaSegmentDescriptor} exists in the underlying database. - */ - public MediaSegmentDescriptor(String objectId, String segmentId, int segmentNumber, int start, int end, float startabs, float endabs, boolean exists) { - this.segmentId = segmentId; - this.objectId = objectId; - this.number = segmentNumber; - this.start = start; - this.end = end; - this.startabs = startabs; - this.endabs = endabs; - this.exists = exists; - } - - public MediaSegmentDescriptor() { - this("", "", 0, 0, 0, 0.0f, 0.0f, false); - } - - @JsonProperty - public String getSegmentId() { - return this.segmentId; - } - - @JsonProperty - public String getObjectId() { - return this.objectId; - } - - /** - * Note: this property is called **segmentnumber** in the data model now. - */ - @JsonProperty - public int getSequenceNumber() { - return this.number; - } - - @JsonProperty - public int getCount() { - return end - start + 1; - } - - @JsonProperty - public int getStart() { - return start; - } - - @JsonProperty - public int getEnd() { - return end; - } - - @JsonProperty - public float getStartabs() { - return this.startabs; - } - - @JsonProperty - public float getEndabs() { - return this.endabs; - } - - @JsonIgnore - @Override - public boolean exists() { - return this.exists; - } - - @Override - public String toString() { - return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); - } + + /** + * Name of the entity in the persistence layer. + */ + public static final String ENTITY = "cineast_segment"; + + /** + * Name of the column with the segment number + */ + public static final String SEGMENT_NO_COL_NAME = "segmentnumber"; + /** + * Name of the column with segmentstart value + */ + public static final String SEGMENT_START_COL_NAME = "segmentstart"; + /** + * Name of the column with segmentend value + */ + public static final String SEGMENT_END_COL_NAME = "segmentend"; + /** + * Name of the column with segmetnstartabs value + */ + public static final String SEGMENT_STARTABS_COL_NAME = "segmentstartabs"; + /** + * Name of the column with segmentendabs value + */ + public static final String SEGMENT_ENDABS_COL_NAME = "segmentendabs"; + + /** + * Field names in the persistence layer. + *

    + * Order (Important): - segmentid (PK) - objectid (FK -> MediaObject) - segmentnumber - segmentstart - segmentend - startabs + */ + public static final String[] FIELDNAMES = { + SEGMENT_ID_COLUMN_QUALIFIER, + OBJECT_ID_COLUMN_QUALIFIER, + SEGMENT_NO_COL_NAME, + SEGMENT_START_COL_NAME, + SEGMENT_END_COL_NAME, + SEGMENT_STARTABS_COL_NAME, + SEGMENT_ENDABS_COL_NAME}; + + private final String segmentId, objectId; + private final int start, end, number; + private final float startabs, endabs; + private final boolean exists; + + /** + * Constructor for {@link MediaSegmentDescriptor}. + * + * @param objectId ID of the {@link MediaObjectDescriptor} this {@link MediaSegmentDescriptor} belongs to. + * @param segmentId ID of the {@link MediaSegmentDescriptor}. + * @param segmentNumber Relative position of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} (starts with 1) + * @param start Start of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in frames (e.g. for videos or audio). + * @param end End of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in frames (e.g. for videos or audio). + * @param startabs Absolute start of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in seconds (e.g. for videos or audio). + * @param endabs Absolute end of the {@link MediaSegmentDescriptor} within the {@link MediaObjectDescriptor} in seconds (e.g. for videos or audio). + * @param exists Whether or not this {@link MediaSegmentDescriptor} exists in the underlying database. + */ + public MediaSegmentDescriptor(String objectId, String segmentId, int segmentNumber, int start, int end, float startabs, float endabs, boolean exists) { + this.segmentId = segmentId; + this.objectId = objectId; + this.number = segmentNumber; + this.start = start; + this.end = end; + this.startabs = startabs; + this.endabs = endabs; + this.exists = exists; + } + + public MediaSegmentDescriptor() { + this("", "", 0, 0, 0, 0.0f, 0.0f, false); + } + + @JsonProperty + public String getSegmentId() { + return this.segmentId; + } + + @JsonProperty + public String getObjectId() { + return this.objectId; + } + + /** + * Note: this property is called **segmentnumber** in the data model now. + */ + @JsonProperty + public int getSequenceNumber() { + return this.number; + } + + @JsonProperty + public int getCount() { + return end - start + 1; + } + + @JsonProperty + public int getStart() { + return start; + } + + @JsonProperty + public int getEnd() { + return end; + } + + @JsonProperty + public float getStartabs() { + return this.startabs; + } + + @JsonProperty + public float getEndabs() { + return this.endabs; + } + + @JsonIgnore + @Override + public boolean exists() { + return this.exists; + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java index 3c0677b07..6de3ceb27 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/SimpleFeatureDescriptor.java @@ -4,21 +4,27 @@ public class SimpleFeatureDescriptor { - /** */ - public final String segmentId; - - /** */ - public final ReadableFloatVector feature; - - public SimpleFeatureDescriptor(String segmentId, ReadableFloatVector feature) { - this.segmentId = segmentId; - this.feature = feature; - } - - public String getSegmentId() { - return segmentId; - } - public ReadableFloatVector getFeature() { - return feature; - } + + /** + * + */ + public final String segmentId; + + /** + * + */ + public final ReadableFloatVector feature; + + public SimpleFeatureDescriptor(String segmentId, ReadableFloatVector feature) { + this.segmentId = segmentId; + this.feature = feature; + } + + public String getSegmentId() { + return segmentId; + } + + public ReadableFloatVector getFeature() { + return feature; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/TagInstance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/TagInstance.java index 9485c145d..c0252d6b2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/TagInstance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/TagInstance.java @@ -10,6 +10,6 @@ public class TagInstance { public TagInstance(String id, Tag tag) { this.id = id; this.tag = tag; -} + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java index 16859943c..927fc29c6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioDescriptor.java @@ -1,81 +1,86 @@ package org.vitrivr.cineast.core.data.frames; /** - * The class encapsulates descriptive information concerning an audio-stream that does not change between frames. The intention behind this - * class is that {@link AudioFrame}s that belong together share the same instance of the AudioDescriptor. - * + * The class encapsulates descriptive information concerning an audio-stream that does not change between frames. The intention behind this class is that {@link AudioFrame}s that belong together share the same instance of the AudioDescriptor. */ public class AudioDescriptor { - /** Samplingrate of the audio associated with this descriptor. */ - private final float samplingrate; - /** Number of channels in the audio associated with this descriptor. */ - private final int channels; + /** + * Samplingrate of the audio associated with this descriptor. + */ + private final float samplingrate; - /** Duration of the audio associated with this descriptor in milliseconds. */ - private final long duration; + /** + * Number of channels in the audio associated with this descriptor. + */ + private final int channels; - /** - * Constructor for an AudioDescriptor. - */ - public AudioDescriptor(float samplingrate, int channels, long duration) { - this.samplingrate = samplingrate; - this.channels = channels; - this.duration = duration; - } + /** + * Duration of the audio associated with this descriptor in milliseconds. + */ + private final long duration; - /** - * Getter for the samplingrate. - * - * @return Samplingrate of the source stream. - */ - public final float getSamplingrate() { - return this.samplingrate; - } + /** + * Constructor for an AudioDescriptor. + */ + public AudioDescriptor(float samplingrate, int channels, long duration) { + this.samplingrate = samplingrate; + this.channels = channels; + this.duration = duration; + } - /** - * Getter for channels. - * - * @return Number of channels in the source stream - */ - public final int getChannels() { - return this.channels; - } - - /** - * Getter for duration. - * - * @return Duration of the total source stream - */ - public final long getDuration() { - return this.duration; - } + /** + * Getter for the samplingrate. + * + * @return Samplingrate of the source stream. + */ + public final float getSamplingrate() { + return this.samplingrate; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + /** + * Getter for channels. + * + * @return Number of channels in the source stream + */ + public final int getChannels() { + return this.channels; + } - AudioDescriptor that = (AudioDescriptor) o; + /** + * Getter for duration. + * + * @return Duration of the total source stream + */ + public final long getDuration() { + return this.duration; + } - if (Float.compare(that.samplingrate, samplingrate) != 0) { - return false; - } - if (channels != that.channels) { - return false; - } - return duration == that.duration; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AudioDescriptor that = (AudioDescriptor) o; - @Override - public int hashCode() { - int result = (samplingrate != +0.0f ? Float.floatToIntBits(samplingrate) : 0); - result = 31 * result + channels; - result = 31 * result + (int) (duration ^ (duration >>> 32)); - return result; + if (Float.compare(that.samplingrate, samplingrate) != 0) { + return false; } + if (channels != that.channels) { + return false; + } + return duration == that.duration; + } + + @Override + public int hashCode() { + int result = (samplingrate != +0.0f ? Float.floatToIntBits(samplingrate) : 0); + result = 31 * result + channels; + result = 31 * result + (int) (duration ^ (duration >>> 32)); + return result; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioFrame.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioFrame.java index fde05c100..11c2e863f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioFrame.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/AudioFrame.java @@ -1,290 +1,294 @@ package org.vitrivr.cineast.core.data.frames; -import javax.sound.sampled.AudioFormat; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import javax.sound.sampled.AudioFormat; /** - * Represents a single audio-frame containing a specific number of samples (the number depends on the decoder that - * created the AudioFrame). Sample data is stored in a byte array and internally represented as 16bit int PCM i.e. each sample - * is represented by a signed 16bit short between -32767 and 32767. - * - * The AudioFrame class supports different sample-rates and an arbitrary number of samples and is compatible with the - * Java Audio API. - * + * Represents a single audio-frame containing a specific number of samples (the number depends on the decoder that created the AudioFrame). Sample data is stored in a byte array and internally represented as 16bit int PCM i.e. each sample is represented by a signed 16bit short between -32767 and 32767. + *

    + * The AudioFrame class supports different sample-rates and an arbitrary number of samples and is compatible with the Java Audio API. */ public class AudioFrame { - /** Default empty frames frame. Encodes a single, mute sample for one channel. */ - public final static AudioFrame EMPTY_FRAME = new AudioFrame(0,0, new byte[0], new AudioDescriptor(22050, 1, 0)); - - /** Number of bits in a sample. */ - public final static int BITS_PER_SAMPLE = 16; - - /** Index of the AudioFrame usually generated in the decoding context (e.g. i-th frame of the decoded file). */ - private final long idx; - - /** Timestamp in milliseconds of the first sample in the AudioFrame, relative to the whole file. */ - private final long timestamp; - - /** AudioDescriptor that describes the audio in this frame. */ - private final AudioDescriptor descriptor; - - /** Number of samples per channel in this AudioFrame. */ - private int numberOfSamples; - - /** ByteBuffer holding the raw 16bit int data. */ - private ByteBuffer data; - - /** - * Default constructor. - * - * @param idx Index of the first sample (pair) in the AudioFrame. - * @param timestamp Index of the first sample. - * @param descriptor AudioDescriptor for the stream this frame stems from. - * @param data Byte array containing 16bit signed PCM data. - */ - public AudioFrame(long idx, long timestamp, byte[] data, AudioDescriptor descriptor) { - this.idx = idx; - this.descriptor = descriptor; - this.timestamp = timestamp; - this.setData(data); - } - public AudioFrame(AudioFrame other){ - this(other.idx, other.timestamp, other.data.array(), new AudioDescriptor(other.descriptor.getSamplingrate(), other.descriptor.getChannels(), other.descriptor.getDuration())); + /** + * Default empty frames frame. Encodes a single, mute sample for one channel. + */ + public final static AudioFrame EMPTY_FRAME = new AudioFrame(0, 0, new byte[0], new AudioDescriptor(22050, 1, 0)); + + /** + * Number of bits in a sample. + */ + public final static int BITS_PER_SAMPLE = 16; + + /** + * Index of the AudioFrame usually generated in the decoding context (e.g. i-th frame of the decoded file). + */ + private final long idx; + + /** + * Timestamp in milliseconds of the first sample in the AudioFrame, relative to the whole file. + */ + private final long timestamp; + + /** + * AudioDescriptor that describes the audio in this frame. + */ + private final AudioDescriptor descriptor; + + /** + * Number of samples per channel in this AudioFrame. + */ + private int numberOfSamples; + + /** + * ByteBuffer holding the raw 16bit int data. + */ + private ByteBuffer data; + + /** + * Default constructor. + * + * @param idx Index of the first sample (pair) in the AudioFrame. + * @param timestamp Index of the first sample. + * @param descriptor AudioDescriptor for the stream this frame stems from. + * @param data Byte array containing 16bit signed PCM data. + */ + public AudioFrame(long idx, long timestamp, byte[] data, AudioDescriptor descriptor) { + this.idx = idx; + this.descriptor = descriptor; + this.timestamp = timestamp; + this.setData(data); + } + + public AudioFrame(AudioFrame other) { + this(other.idx, other.timestamp, other.data.array(), new AudioDescriptor(other.descriptor.getSamplingrate(), other.descriptor.getChannels(), other.descriptor.getDuration())); + } + + /** + * Returns a Java AudioFormat object that specifies the arrangement of the frames-data in the current AudioFrame. + * + * @return AudioFormat + */ + public final AudioFormat getFormat() { + return new AudioFormat(this.descriptor.getSamplingrate(), BITS_PER_SAMPLE, this.descriptor.getChannels(), true, false); + } + + /** + * Returns the AudioDescriptor associated with the this AudioFrame. + * + * @return AudioDescriptor + */ + public final AudioDescriptor getDescriptor() { + return this.descriptor; + } + + /** + * Returns the presentation timestamp of the first sample. + * + * @return Presentation timestamp pf the first sample. + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * Returns the size of the {@link AudioFrame} data in bytes. + * + * @return Size of {@link AudioFrame} in bytes. + */ + public final int size() { + return this.data.array().length; + } + + /** + * Returns the total number of samples per channel in this {@link AudioFrame}. + * + * @return Number of samples per channel. + */ + public final int numberOfSamples() { + return this.numberOfSamples; + } + + /** + * Getter for the frame id (counter). + * + * @return Sample id + */ + public final long getIdx() { + return this.idx; + } + + /** + * Getter for the raw byte array. + * + * @return Byte array containing the frames data of this AudioFrame. + */ + public final byte[] getData() { + return this.data.array(); + } + + /** + * Getter for sample-rate. + * + * @return Sample rate of this AudioFrame. + */ + public final float getSamplingrate() { + return this.descriptor.getSamplingrate(); + } + + /** + * Returns the duration of the {@link AudioFrame} in seconds. + * + * @return Duration of the {@link AudioFrame} + */ + public final float getDuration() { + return this.numberOfSamples / this.descriptor.getSamplingrate(); + } + + /** + * Returns the relative start of the {@link AudioFrame} in seconds. + * + * @return Relative start of the {@link AudioFrame} + */ + public final float getStart() { + return this.timestamp / 1000.0f; + } + + /** + * Returns the relative end of the AudioFrame in seconds. + * + * @return Relative end of the {@link AudioFrame}. + */ + public final float getEnd() { + return this.getStart() + this.numberOfSamples / this.descriptor.getSamplingrate(); + } + + /** + * Getter for the number of channels. + * + * @return Number of channels in this AudioFrame. + */ + public final int getChannels() { + return this.descriptor.getChannels(); + } + + /** + * Returns the sample specified sample in the specified channel as short value. + * + * @param idx Index of the sample (zero-based) + * @param channel Index of the channel (zero-based) + * @return Sample value for the specified channel at the specified index. + */ + public final short getSampleAsShort(int idx, int channel) { + if (channel < this.descriptor.getChannels()) { + return this.data.getShort(2 * idx * this.descriptor.getChannels() + 2 * channel); + } else { + throw new IllegalArgumentException("The channel indexed must not exceed the number of channels!"); } - - /** - * Returns a Java AudioFormat object that specifies the arrangement of the frames-data in the - * current AudioFrame. - * - * @return AudioFormat - */ - public final AudioFormat getFormat() { - return new AudioFormat(this.descriptor.getSamplingrate(), BITS_PER_SAMPLE, this.descriptor.getChannels(), true, false); + } + + /** + * Returns the sample specified sample in the specified channel as double value between -1.0 and 1.0. + * + * @param idx Index of the sample (zero-based) + * @param channel Index of the channel (zero-based) + * @return Sample value for the specified channel at the specified index. + */ + public final double getSampleAsDouble(int idx, int channel) { + return ((double) this.getSampleAsShort(idx, channel) / (double) Short.MAX_VALUE); + } + + /** + * Calculates and returns the mean sample value (across all channels) at the specified sample index and returns it as short value. + * + * @param idx Index of the sample (zero-based) + * @return Mean value of the sample at the specified index. + */ + public final short getMeanSampleAsShort(int idx) { + int meanSample = 0; + for (int i = 0; i < this.descriptor.getChannels(); i++) { + meanSample += this.getSampleAsShort(idx, i); } - - /** - * Returns the AudioDescriptor associated with the this AudioFrame. - * - * @return AudioDescriptor - */ - public final AudioDescriptor getDescriptor() { - return this.descriptor; + return (short) (meanSample / this.descriptor.getChannels()); + } + + /** + * Calculates and returns the mean sample value (across all channels) at the specified sample index and returns it as double value between -1.0 and 1.0 + * + * @param idx Index of the sample (zero-based) + * @return Mean value of the sample at the specified index as float. + */ + public final double getMeanSampleAsDouble(int idx) { + float meanSample = 0; + for (int i = 0; i < this.descriptor.getChannels(); i++) { + meanSample += this.getSampleAsShort(idx, i); } - - /** - * Returns the presentation timestamp of the first sample. - * - * @return Presentation timestamp pf the first sample. - */ - public long getTimestamp() { - return this.timestamp; + return (meanSample / (this.descriptor.getChannels() * Short.MAX_VALUE)); + } + + /** + * Appends an AudioFrame to the current AudioFrame if the two frames have the same specs in terms of sampleRate and number of channels. The raw bytes of the other AudioFrame are appended to the byte-array of the current AudioFrame. + * + * @param that The AudioFrame to append to the current frame. + * @param numberOfSamples The number of samples to append. Must be smaller than the size of the other AudioFrame! + * @return true if appending was successful, false otherwise. + */ + public boolean append(AudioFrame that, int numberOfSamples) { + if (!this.descriptor.equals(that.descriptor)) { + return false; } - - /** - * Returns the size of the {@link AudioFrame} data in bytes. - * - * @return Size of {@link AudioFrame} in bytes. - */ - public final int size() { - return this.data.array().length; + int bytes = that.descriptor.getChannels() * numberOfSamples * (BITS_PER_SAMPLE / 8); + if (bytes > that.data.capacity()) { + return false; } - /** - * Returns the total number of samples per channel in this {@link AudioFrame}. - * - * @return Number of samples per channel. - */ - public final int numberOfSamples() { - return this.numberOfSamples; + /* Copy data. */ + byte[] data = new byte[this.data.capacity() + bytes]; + System.arraycopy(this.data.array(), 0, data, 0, this.data.capacity()); + System.arraycopy(that.data.array(), 0, data, this.data.capacity(), bytes); + + /* Update local ByteBuffer reference. */ + this.setData(data); + return true; + } + + /** + * Appends the provided {@link AudioFrame} to the this {@link AudioFrame} and returns true, if the operation was successful and false otherwise. + * + * @param that The {@link AudioFrame} that should be appended. + * @return True on success, false otherwise. + */ + public boolean append(AudioFrame that) { + return this.append(that, that.numberOfSamples); + } + + public AudioFrame split(int numberOfSamples) { + + if (numberOfSamples > this.numberOfSamples) { + return this; } - /** - * Getter for the frame id (counter). - * - * @return Sample id - */ - public final long getIdx() { - return this.idx; - } + int bytesToCut = this.descriptor.getChannels() * numberOfSamples * (BITS_PER_SAMPLE / 8); + byte[] cutBytes = new byte[bytesToCut]; + byte[] remaining = new byte[this.data.capacity() - bytesToCut]; - /** - * Getter for the raw byte array. - * - * @return Byte array containing the frames data of this AudioFrame. - */ - public final byte[] getData() { - return this.data.array(); - } + System.arraycopy(this.data.array(), 0, cutBytes, 0, bytesToCut); + System.arraycopy(this.data.array(), bytesToCut, remaining, 0, remaining.length); - /** - * Getter for sample-rate. - * - * @return Sample rate of this AudioFrame. - */ - public final float getSamplingrate() { - return this.descriptor.getSamplingrate(); - } + setData(remaining); - /** - * Returns the duration of the {@link AudioFrame} in seconds. - * - * @return Duration of the {@link AudioFrame} - */ - public final float getDuration() { - return this.numberOfSamples/this.descriptor.getSamplingrate(); - } - - /** - * Returns the relative start of the {@link AudioFrame} in seconds. - * - * @return Relative start of the {@link AudioFrame} - */ - public final float getStart() { - return this.timestamp/1000.0f; - } - - /** - * Returns the relative end of the AudioFrame in seconds. - * - * @return Relative end of the {@link AudioFrame}. - */ - public final float getEnd() { - return this.getStart() + this.numberOfSamples/this.descriptor.getSamplingrate(); - } - - /** - * Getter for the number of channels. - * - * @return Number of channels in this AudioFrame. - */ - public final int getChannels() { - return this.descriptor.getChannels(); - } - - /** - * Returns the sample specified sample in the specified channel as short value. - * - * @param idx Index of the sample (zero-based) - * @param channel Index of the channel (zero-based) - * @return Sample value for the specified channel at the specified index. - */ - public final short getSampleAsShort(int idx, int channel) { - if (channel < this.descriptor.getChannels()) { - return this.data.getShort(2*idx * this.descriptor.getChannels() + 2*channel); - } else { - throw new IllegalArgumentException("The channel indexed must not exceed the number of channels!"); - } - } + return new AudioFrame(idx, timestamp, cutBytes, new AudioDescriptor(descriptor.getSamplingrate(), descriptor.getChannels(), (long) (numberOfSamples / descriptor.getSamplingrate()))); + } - /** - * Returns the sample specified sample in the specified channel as double - * value between -1.0 and 1.0. - * - * @param idx Index of the sample (zero-based) - * @param channel Index of the channel (zero-based) - * @return Sample value for the specified channel at the specified index. - */ - public final double getSampleAsDouble(int idx, int channel) { - return ((double)this.getSampleAsShort(idx, channel)/(double)Short.MAX_VALUE); - } - - /** - * Calculates and returns the mean sample value (across all channels) - * at the specified sample index and returns it as short value. - * - * @param idx Index of the sample (zero-based) - * @return Mean value of the sample at the specified index. - */ - public final short getMeanSampleAsShort(int idx) { - int meanSample = 0; - for (int i=0;i that.data.capacity()) { - return false; - } - - /* Copy data. */ - byte[] data = new byte[this.data.capacity() + bytes]; - System.arraycopy(this.data.array(), 0, data, 0, this.data.capacity()); - System.arraycopy(that.data.array(), 0, data, this.data.capacity(), bytes); - - /* Update local ByteBuffer reference. */ - this.setData(data); - return true; - } - - /** - * Appends the provided {@link AudioFrame} to the this {@link AudioFrame} and returns true, if the - * operation was successful and false otherwise. - * - * @param that The {@link AudioFrame} that should be appended. - * @return True on success, false otherwise. - */ - public boolean append(AudioFrame that) { - return this.append(that, that.numberOfSamples); - } - - public AudioFrame split(int numberOfSamples){ - - if (numberOfSamples > this.numberOfSamples){ - return this; - } - - int bytesToCut = this.descriptor.getChannels() * numberOfSamples * (BITS_PER_SAMPLE/8); - byte[] cutBytes = new byte[bytesToCut]; - byte[] remaining = new byte[this.data.capacity() - bytesToCut]; - - System.arraycopy(this.data.array(), 0, cutBytes, 0, bytesToCut); - System.arraycopy(this.data.array(), bytesToCut, remaining, 0, remaining.length); - - setData(remaining); - - return new AudioFrame(idx, timestamp, cutBytes, new AudioDescriptor(descriptor.getSamplingrate(), descriptor.getChannels(), (long) (numberOfSamples/descriptor.getSamplingrate()))); - } - - /** - * Internal method to update the buffer holding the actual audio data. - * - * @param data Byte array with the samples. - */ - private void setData(byte[] data) { - this.data = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); - this.numberOfSamples = data.length/(2 * this.descriptor.getChannels()); - } + /** + * Internal method to update the buffer holding the actual audio data. + * + * @param data Byte array with the samples. + */ + private void setData(byte[] data) { + this.data = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + this.numberOfSamples = data.length / (2 * this.descriptor.getChannels()); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java index 462b3f36e..88a1b31f0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoDescriptor.java @@ -1,78 +1,85 @@ package org.vitrivr.cineast.core.data.frames; /** - * The class encapsulates descriptive information concerning a video-stream (visual only) that does not change between frames. The intention behind this - * class is that {@link VideoFrame}s that belong together share the same instance of the AudioDescriptor. - * + * The class encapsulates descriptive information concerning a video-stream (visual only) that does not change between frames. The intention behind this class is that {@link VideoFrame}s that belong together share the same instance of the AudioDescriptor. */ public class VideoDescriptor { - /** Frame rate of the video associated with this descriptor. */ - private final float fps; - /** Duration of the video associated with this descriptor in milliseconds. */ - private final long duration; + /** + * Frame rate of the video associated with this descriptor. + */ + private final float fps; - /** Width of the video associated with this descriptor. */ - private final int width; + /** + * Duration of the video associated with this descriptor in milliseconds. + */ + private final long duration; - /** Height of the video associated with this descriptor. */ - private final int height; + /** + * Width of the video associated with this descriptor. + */ + private final int width; - /** - * Constructor for VideoDescriptor - */ - public VideoDescriptor(float fps, long duration, int width, int height) { - this.fps = fps; - this.duration = duration; - this.width = width; - this.height = height; - } - - public float getFps() { - return fps; - } + /** + * Height of the video associated with this descriptor. + */ + private final int height; - public long getDuration() { - return duration; - } + /** + * Constructor for VideoDescriptor + */ + public VideoDescriptor(float fps, long duration, int width, int height) { + this.fps = fps; + this.duration = duration; + this.width = width; + this.height = height; + } - public int getWidth() { - return width; - } + public float getFps() { + return fps; + } - public int getHeight() { - return height; - } + public long getDuration() { + return duration; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + public int getWidth() { + return width; + } - VideoDescriptor that = (VideoDescriptor) o; + public int getHeight() { + return height; + } - if (Float.compare(that.fps, fps) != 0) { - return false; - } - if (duration != that.duration) { - return false; - } - if (width != that.width) { - return false; - } - return height == that.height; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VideoDescriptor that = (VideoDescriptor) o; - @Override - public int hashCode() { - int result = (fps != +0.0f ? Float.floatToIntBits(fps) : 0); - result = 31 * result + (int) (duration ^ (duration >>> 32)); - result = 31 * result + width; - result = 31 * result + height; - return result; + if (Float.compare(that.fps, fps) != 0) { + return false; + } + if (duration != that.duration) { + return false; } + if (width != that.width) { + return false; + } + return height == that.height; + } + + @Override + public int hashCode() { + int result = (fps != +0.0f ? Float.floatToIntBits(fps) : 0); + result = 31 * result + (int) (duration ^ (duration >>> 32)); + result = 31 * result + width; + result = 31 * result + height; + return result; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoFrame.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoFrame.java index 1a02bf7cb..79c401888 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoFrame.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/frames/VideoFrame.java @@ -1,151 +1,167 @@ package org.vitrivr.cineast.core.data.frames; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; -import java.util.*; - /** - * Represents a single video-frame. Such a frame consist of a single image and, optionally, an {@link AudioFrame} of arbitrary length and, - * again optionally, a list of {@link SubtitleItem}s. + * Represents a single video-frame. Such a frame consist of a single image and, optionally, an {@link AudioFrame} of arbitrary length and, again optionally, a list of {@link SubtitleItem}s. * * @see AudioFrame */ public class VideoFrame { - public static final VideoFrame EMPTY_VIDEO_FRAME = new VideoFrame(0, 0, MultiImage.EMPTY_MULTIIMAGE, new VideoDescriptor(25, 40, 1, 1)); - - /** ID of the VideoFrame. */ - private final int id; - - /** Timestamp in milliseconds of the VideoFrame relative to the whole file. */ - private final long timestamp; - - /** MultiImage representing the current VideoFrame. */ - private MultiImage img; - - /** {@link AudioFrame} that is associated with the current frame. May be null! */ - private AudioFrame audioFrame = null; - - /** {@link SubtitleItem} that is associated with the current video frame. May be null! */ - private List subtitleItems = null; - - /** VideoDescriptor that describes the video this frame belongs to. */ - private final VideoDescriptor descriptor; - - /** - * Constructor vor VideoFrame. - * - * @param id Incremental ID from the frame (e.g. as returned by the decoder). - * @param image Image representing the VideoFrame. - */ - public VideoFrame(int id, long timestamp, MultiImage image, VideoDescriptor descriptor){ - this.id = id; - this.timestamp = timestamp; - this.img = image; - this.descriptor = descriptor; - } - - /** - * Getter for ID. - * - * @return ID of the current video frame. - */ - public int getId(){ - return this.id; - } - - /** - * Getter for {@link VideoDescriptor}. - * - * @return {@link VideoDescriptor} - */ - public VideoDescriptor getDescriptor() { - return descriptor; - } - - /** - * Returns the presentation timestamp of the {@link VideoFrame} in milliseconds. - * - * @return Presentation timestamp of the {@link VideoFrame}. - */ - public long getTimestamp() { - return this.timestamp; - } - - /** - * Returns the presentation timestamp of the {@link VideoFrame} in seconds. - * - * @return Presentation timestamp of the {@link VideoFrame} in seconds. - */ - public float getTimestampSeconds() { - return this.timestamp/1000.0f; - } - - /** - * Getter for frame image. - * - * @return MultiImage representing the current frame. - */ - public MultiImage getImage(){ - return this.img; - } - - /** - * Getter for frame audio. - * - * @return AudioFrame containing the sound of the current frame. - */ - public final Optional getAudio() { - return Optional.ofNullable(this.audioFrame); - } - /** - * Getter for subtitle item. - * - * @return {@link SubtitleItem} associated with current video frame. - */ - public final List getSubtitleItems() { - if (this.subtitleItems != null) { - return Collections.unmodifiableList(this.subtitleItems); - } else { - return Collections.unmodifiableList(new ArrayList<>(0)); - } - } - - /** - * Adds an AudioFrame to the current VideoFrame. The existing frame (if any) - * and the new frame are concatenated during the process. - * - * @param frame AudioFrame to add to this VideoFrame. - */ - public void addAudioFrame(AudioFrame frame) { - if (this.audioFrame == null) { - this.audioFrame = frame; - } else { - this.audioFrame.append(frame); - } + public static final VideoFrame EMPTY_VIDEO_FRAME = new VideoFrame(0, 0, MultiImage.EMPTY_MULTIIMAGE, new VideoDescriptor(25, 40, 1, 1)); + + /** + * ID of the VideoFrame. + */ + private final int id; + + /** + * Timestamp in milliseconds of the VideoFrame relative to the whole file. + */ + private final long timestamp; + + /** + * MultiImage representing the current VideoFrame. + */ + private MultiImage img; + + /** + * {@link AudioFrame} that is associated with the current frame. May be null! + */ + private AudioFrame audioFrame = null; + + /** + * {@link SubtitleItem} that is associated with the current video frame. May be null! + */ + private List subtitleItems = null; + + /** + * VideoDescriptor that describes the video this frame belongs to. + */ + private final VideoDescriptor descriptor; + + /** + * Constructor vor VideoFrame. + * + * @param id Incremental ID from the frame (e.g. as returned by the decoder). + * @param image Image representing the VideoFrame. + */ + public VideoFrame(int id, long timestamp, MultiImage image, VideoDescriptor descriptor) { + this.id = id; + this.timestamp = timestamp; + this.img = image; + this.descriptor = descriptor; + } + + /** + * Getter for ID. + * + * @return ID of the current video frame. + */ + public int getId() { + return this.id; + } + + /** + * Getter for {@link VideoDescriptor}. + * + * @return {@link VideoDescriptor} + */ + public VideoDescriptor getDescriptor() { + return descriptor; + } + + /** + * Returns the presentation timestamp of the {@link VideoFrame} in milliseconds. + * + * @return Presentation timestamp of the {@link VideoFrame}. + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * Returns the presentation timestamp of the {@link VideoFrame} in seconds. + * + * @return Presentation timestamp of the {@link VideoFrame} in seconds. + */ + public float getTimestampSeconds() { + return this.timestamp / 1000.0f; + } + + /** + * Getter for frame image. + * + * @return MultiImage representing the current frame. + */ + public MultiImage getImage() { + return this.img; + } + + /** + * Getter for frame audio. + * + * @return AudioFrame containing the sound of the current frame. + */ + public final Optional getAudio() { + return Optional.ofNullable(this.audioFrame); + } + + /** + * Getter for subtitle item. + * + * @return {@link SubtitleItem} associated with current video frame. + */ + public final List getSubtitleItems() { + if (this.subtitleItems != null) { + return Collections.unmodifiableList(this.subtitleItems); + } else { + return Collections.unmodifiableList(new ArrayList<>(0)); } - - /** - * Add a {@link SubtitleItem} to the current {@link VideoFrame}. - * - * @param item New {@link SubtitleItem}. Must not be null. - */ - public void addSubtitleItem(SubtitleItem item) { - if (this.subtitleItems == null) this.subtitleItems = new LinkedList<>(); - if (!this.subtitleItems.contains(item)) { - this.subtitleItems.add(item); - } - } - - /** - * Clears the VideoFrame. - */ - public void clear(){ - this.img.clear(); - this.img = null; - this.audioFrame = null; - } + } + + /** + * Adds an AudioFrame to the current VideoFrame. The existing frame (if any) and the new frame are concatenated during the process. + * + * @param frame AudioFrame to add to this VideoFrame. + */ + public void addAudioFrame(AudioFrame frame) { + if (this.audioFrame == null) { + this.audioFrame = frame; + } else { + this.audioFrame.append(frame); + } + } + + /** + * Add a {@link SubtitleItem} to the current {@link VideoFrame}. + * + * @param item New {@link SubtitleItem}. Must not be null. + */ + public void addSubtitleItem(SubtitleItem item) { + if (this.subtitleItems == null) { + this.subtitleItems = new LinkedList<>(); + } + if (!this.subtitleItems.contains(item)) { + this.subtitleItems.add(item); + } + } + + /** + * Clears the VideoFrame. + */ + public void clear() { + this.img.clear(); + this.img = null; + this.audioFrame = null; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java index baaeeab65..15666df0d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/Mesh.java @@ -1,566 +1,594 @@ package org.vitrivr.cineast.core.data.m3d; -import org.apache.commons.math3.util.FastMath; -import org.joml.*; -import org.vitrivr.cineast.core.util.mesh.MeshMathUtil; - -import java.awt.*; +import java.awt.Color; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.commons.math3.util.FastMath; +import org.joml.Matrix4f; +import org.joml.Vector3f; +import org.joml.Vector3fc; +import org.joml.Vector3i; +import org.joml.Vector4i; +import org.vitrivr.cineast.core.util.mesh.MeshMathUtil; public class Mesh implements WritableMesh { - /** The default, empty mesh. */ - public static final Mesh EMPTY = new Mesh(1,1); + /** + * The default, empty mesh. + */ + public static final Mesh EMPTY = new Mesh(1, 1); + + /** + * A single vertex, which contains all the information about position, normal and color. + */ + public class Vertex { /** - * A single vertex, which contains all the information about position, normal and color. + * Position of the vertex in 3D space. */ - public class Vertex { - /** Position of the vertex in 3D space. */ - private final Vector3f position; - - /** The vertex-normal. */ - private final Vector3f normal; - - /** Color of the vertex. */ - private final Vector3f color; - - /** List of faces the current vertex participates in. */ - private final List faces = new ArrayList<>(4); - - public Vertex(Vector3f position) { - this(position, new Vector3f(0.0f, 0.0f, 0.0f), new Vector3f(0.0f, 0.0f, 0.0f)); - } - - public Vertex(Vector3f position,Vector3f color, Vector3f normal) { - this.position = position; - this.normal = normal; - this.color = color; - } - - /** - * Returns the number of faces this vertex participates in. - * - * @return Number of faces. - */ - public int numberOfFaces() { - return this.faces.size(); - } - - - /** - * Getter to the position vector. - * - * @return Immutable version of the position vector. - */ - public Vector3fc getPosition() { - return this.position; - } - - /** - * Getter to the vertex normal. - * - * @return Immutable version of the vertex normal. - */ - public Vector3fc getNormal() { - return this.normal; - } - - /** - * Getter to the vertex color. - * - * @return Immutable version of the vertex normal. - */ - public Vector3fc getColor() { - return this.color; - } - - /** - * Attaches the vertex to a face, which causes the vertex-normal - * to be re-calculated. - * - * @param face Face to which the vertex should be attached. - */ - private void attachToFace(Face face) { - if (!this.faces.contains(face)) { - this.faces.add(face); - this.rebuild(); - } - } - - /** - * Detaches the vertex from a face, which causes the vertex-normal - * to be re-calculated. - * - * @param face Face to which the vertex should be attached. - */ - private void detachFromFace(Face face) { - if (!this.faces.contains(face)) { - this.faces.remove(face); - this.rebuild(); - } - } - - /** - * Re-calculates the vertex-normal by calculating the weighted mean of all - * face-normals this vertex participates in. - */ - private void rebuild() { - this.normal.x = 0.0f; - this.normal.y = 0.0f; - this.normal.z = 0.0f; - - for (Face face : this.faces) { - Vector3f fn = face.normal(); - if (!Float.isNaN(fn.x) && !Float.isNaN(fn.y) && !Float.isNaN(fn.z)) { - this.normal.x += fn.x / this.numberOfFaces(); - this.normal.y += fn.y / this.numberOfFaces(); - this.normal.z += fn.z / this.numberOfFaces(); - } - } - } - } + private final Vector3f position; /** - * A face defined that is made up by either three or four vertices. + * The vertex-normal. */ - public class Face { - /** Type of face, defaults to TRI (= three vertices). */ - private final FaceType type; - - /** List of vertices in this face. */ - private final Vertex[] vertices; - - /** Vertex indices. */ - private final int[] vertexIndices; - - /** - * Getter for the face's type. - * - * @return Type of the face. - */ - public final FaceType getType() { - return type; - } - - /** - * Constructor for a face. - */ - private Face(Vector4i indices) { - /* If the w-index is greater than -1 a QUAD face is created. */ - if (indices.w > -1) { - this.type = FaceType.QUAD; - this.vertices = new Vertex[4]; - this.vertexIndices = new int[4]; - } else { - this.type = FaceType.TRI; - this.vertices = new Vertex[3]; - this.vertexIndices = new int[3]; - } - - /* Store vertex-indices. */ - this.vertexIndices[0] = indices.x; - this.vertexIndices[1] = indices.y; - this.vertexIndices[2] = indices.z; - if (this.getType() == FaceType.QUAD) { - this.vertexIndices[3] = indices.w; - } - - /* Add vertices to face. */ - this.vertices[0] = Mesh.this.vertices.get(this.vertexIndices[0]); - this.vertices[1] = Mesh.this.vertices.get(this.vertexIndices[1]); - this.vertices[2] = Mesh.this.vertices.get(this.vertexIndices[2]); - if (this.getType() == FaceType.QUAD) { - this.vertices[3] = Mesh.this.vertices.get(this.vertexIndices[3]); - } - - /* Attach face to vertices. */ - this.vertices[0].attachToFace(this); - this.vertices[1].attachToFace(this); - this.vertices[2].attachToFace(this); - if (this.getType() == FaceType.QUAD) { - this.vertices[3].attachToFace(this); - } - } - - /** - * Returns the list of vertices that make up this face. - * - * @return Unmodifiable list of vertices. - */ - public final List getVertices() { - return Arrays.asList(this.vertices); - } - - /** - * Calculates and returns the area of a face. - * - * @return Area of the face. - */ - public double area() { - if (this.type == FaceType.TRI) { - /* Extract vertices. */ - Vector3f v1 = this.vertices[0].position; - Vector3f v2 = this.vertices[1].position; - Vector3f v3 = this.vertices[2].position; - - /* Generate the edges and sort them in ascending order. */ - List edges = new ArrayList<>(); - edges.add(new Vector3f(v1).sub(v2)); - edges.add(new Vector3f(v2).sub(v3)); - edges.add(new Vector3f(v3).sub(v1)); - - edges.sort((o1, o2) -> { - float difference = o1.length() - o2.length(); - if (difference < 0) { - return -1; - } else if (difference > 0) { - return 1; - } else { - return 0; - } - }); - - float a = edges.get(2).length(); - float b = edges.get(1).length(); - float c = edges.get(0).length(); - - /* Returns the area of the triangle according to Heron's Formula. */ - double area = 0.25 * FastMath.sqrt((a+(b+c)) * (c-(a-b)) * (c+(a-b)) * (a+(b-c))); - if (Double.isNaN(area)) { - return 0.0f; - } else { - return area; - } - } else { - /* Extract vertices. */ - Vector3f v1 = this.vertices[0].position; - Vector3f v2 = this.vertices[1].position; - Vector3f v3 = this.vertices[2].position; - Vector3f v4 = this.vertices[3].position; - - /* Calculates the area of the face using Bretschneider's Formula. */ - Vector3f s1 = new Vector3f(v1).sub(v2); - Vector3f s2 = new Vector3f(v2).sub(v3); - Vector3f s3 = new Vector3f(v3).sub(v4); - Vector3f s4 = new Vector3f(v4).sub(v1); - - Vector3f d1 = new Vector3f(v1).sub(v3); - Vector3f d2 = new Vector3f(v2).sub(v4); - return 0.25 * FastMath.sqrt(4*FastMath.pow(d1.length(),2)*FastMath.pow(d2.length(),2) - FastMath.pow((FastMath.pow(s2.length(),2) + FastMath.pow(s4.length(),2)-FastMath.pow(s1.length(),2)-FastMath.pow(s3.length(),2)),2)); - } - } - - /** - * Calculates and returns the face normal. - */ - public Vector3f normal() { - Vector3f e1 = new Vector3f(this.vertices[1].position).sub(this.vertices[0].position); - Vector3f e2 = new Vector3f(this.vertices[2].position).sub(this.vertices[0].position); - return e1.cross(e2).normalize(); - } - - /** - * Calculates and returns the centroid of the face. - * - * @return Centroid of the face. - */ - public Vector3f centroid() { - Vector3f centroid = new Vector3f(0f,0f,0f); - for (Vertex vertex : this.vertices) { - centroid.add(vertex.position); - } - - if (this.type == FaceType.TRI) { - centroid.div(3.0f); - } else { - centroid.div(4.0f); - } - - return centroid; - } - } - - /** Enumeration used to distinguish between triangular and quadratic faces. */ - public enum FaceType { - TRI,QUAD; - } - - /** List of vertices in the Mesh. */ - private final List vertices; - - /** List of faces in the mesh. */ - private final List faces; - - /** The position of the Mesh's barycenter. Its value is lazily calculated during invocation of the @see barycenter() method. */ - private Vector3f barycenter; - - /** The surface-area of the mesh. Its value is lazily calculated during invocation of the @see surfaceArea() method. */ - private Double surfaceArea; - - /** The bounding box of the mesh. Its value is lazily calculated during invocation of the @see bounds() method. */ - private float[] boundingbox; + private final Vector3f normal; /** - * Copy constructor for Mesh. - * - * @param mesh Mesh that should be copied. + * Color of the vertex. */ - public Mesh(ReadableMesh mesh) { - this(mesh.numberOfFaces(), mesh.numberOfVertices()); - - for (Vertex vertex : mesh.getVertices()) { - this.addVertex(new Vector3f(vertex.position), new Vector3f(vertex.color), new Vector3f(vertex.normal)); - } - - for (Face face : mesh.getFaces()) { - if (face.getType() == FaceType.QUAD) { - this.addFace(new Vector4i(face.vertexIndices[0], face.vertexIndices[1], face.vertexIndices[2], face.vertexIndices[3])); - } else { - this.addFace(new Vector4i(face.vertexIndices[0], face.vertexIndices[1], face.vertexIndices[2], -1)); - } - } - } + private final Vector3f color; /** - * Default constructor. - * - * @param faces Expected number of faces (not a fixed limit). - * @param vertices Expected number of vertices (not a fixed limit). + * List of faces the current vertex participates in. */ - public Mesh(int faces, int vertices) { - this.faces = new ArrayList<>(faces); - this.vertices = new ArrayList<>(vertices); + private final List faces = new ArrayList<>(4); + + public Vertex(Vector3f position) { + this(position, new Vector3f(0.0f, 0.0f, 0.0f), new Vector3f(0.0f, 0.0f, 0.0f)); } - /** - * Adds an vector defining a vertex to the Mesh. - */ - public synchronized void addVertex(Vector3f vertex) { - this.addVertex(vertex, new Vector3f(1.0f, 1.0f, 1.0f)); + public Vertex(Vector3f position, Vector3f color, Vector3f normal) { + this.position = position; + this.normal = normal; + this.color = color; } /** - * Adds a vector defining a vertex to the Mesh. + * Returns the number of faces this vertex participates in. + * + * @return Number of faces. */ - public synchronized void addVertex(Vector3f vertex, Vector3f color) { - this.addVertex(vertex, color, new Vector3f(0.0f, 0.0f, 0.0f)); + public int numberOfFaces() { + return this.faces.size(); } + /** - * Adds an vector defining a vertex to the Mesh. + * Getter to the position vector. + * + * @return Immutable version of the position vector. */ - public synchronized void addVertex(Vector3f vertex, Vector3f color, Vector3f normal) { - this.vertices.add(new Vertex(vertex, color, normal)); + public Vector3fc getPosition() { + return this.position; } /** - * Adds a new triangular face to the Mesh. Faces index vertices and - * vertex normals. + * Getter to the vertex normal. * - * @param vertices Vector3i containing the indices of the vertices. + * @return Immutable version of the vertex normal. */ - @Override - public synchronized boolean addFace(Vector3i vertices) { - int limit = this.vertices.size(); - if (vertices.x < limit && vertices.y < limit && vertices.z < limit) { - Mesh.Face face = new Face(new Vector4i(vertices, -1)); - this.faces.add(face); - return true; - } else { - return false; - } + public Vector3fc getNormal() { + return this.normal; } /** - * Adds a new quadratic face to the Mesh. Faces index vertices and - * vertex normals. + * Getter to the vertex color. * - * @param vertices Vector4i containing the indices of the vertices. + * @return Immutable version of the vertex normal. */ - @Override - public synchronized boolean addFace(Vector4i vertices) { - int limit = this.vertices.size(); - if (vertices.x < limit && vertices.y < limit && vertices.z < limit && vertices.w < limit) { - Mesh.Face face = new Face(vertices); - this.faces.add(face); - return true; - } else { - return false; - } + public Vector3fc getColor() { + return this.color; } /** - * Accessor for an individual vertex. + * Attaches the vertex to a face, which causes the vertex-normal to be re-calculated. * - * @param vertexIndex Index of the vertex that should be returned. - * @return Vertex. + * @param face Face to which the vertex should be attached. */ - @Override - public synchronized Vertex getVertex(int vertexIndex) { - return this.vertices.get(vertexIndex); + private void attachToFace(Face face) { + if (!this.faces.contains(face)) { + this.faces.add(face); + this.rebuild(); + } } /** - * Returns the list of vertices. The returned collection is unmodifiable. + * Detaches the vertex from a face, which causes the vertex-normal to be re-calculated. * - * @return Unmodifiable list of vertices. + * @param face Face to which the vertex should be attached. */ - @Override - public List getVertices() { - return Collections.unmodifiableList(this.vertices); + private void detachFromFace(Face face) { + if (!this.faces.contains(face)) { + this.faces.remove(face); + this.rebuild(); + } } /** - * Returns the list of vertex-normals. The returned collection is unmodifiable. - * - * @return Unmodifiable list of faces. + * Re-calculates the vertex-normal by calculating the weighted mean of all face-normals this vertex participates in. */ - @Override - public List getFaces() { - return Collections.unmodifiableList(faces); + private void rebuild() { + this.normal.x = 0.0f; + this.normal.y = 0.0f; + this.normal.z = 0.0f; + + for (Face face : this.faces) { + Vector3f fn = face.normal(); + if (!Float.isNaN(fn.x) && !Float.isNaN(fn.y) && !Float.isNaN(fn.z)) { + this.normal.x += fn.x / this.numberOfFaces(); + this.normal.y += fn.y / this.numberOfFaces(); + this.normal.z += fn.z / this.numberOfFaces(); + } + } } + } + + /** + * A face defined that is made up by either three or four vertices. + */ + public class Face { /** - * Returns the number of vertices in this Mesh. - * - * @return Number of vertices. + * Type of face, defaults to TRI (= three vertices). */ - @Override - public synchronized final int numberOfVertices() { - return this.vertices.size(); - } + private final FaceType type; /** - * Returns the number of faces in this Mesh. - * - * @return Number of faces. + * List of vertices in this face. */ - @Override - public synchronized final int numberOfFaces() { - return this.faces.size(); - } + private final Vertex[] vertices; /** - * Indicates, whether the mesh is an empty Mesh or not - * - * @return True if mesh is empty, false otherwise. + * Vertex indices. */ - @Override - public synchronized final boolean isEmpty() { - return this.faces.isEmpty(); - } + private final int[] vertexIndices; /** - * Returns the total surface area of the Mesh. + * Getter for the face's type. * - * @return Surface area of the mesh. + * @return Type of the face. */ - @Override - public synchronized final double surfaceArea() { - if (this.surfaceArea == null) { - this.surfaceArea = 0.0; - for (Face face : this.faces) { - this.surfaceArea += face.area(); - } - } - return this.surfaceArea; + public final FaceType getType() { + return type; } /** - * Calculates and returns the Mesh's bounding-box - * - * @return Bounding-box of the mesh. + * Constructor for a face. */ - @Override - public synchronized float[] bounds() { - if (this.boundingbox == null) { - this.boundingbox = MeshMathUtil.bounds(this); - } - return Arrays.copyOf(this.boundingbox, 6); + private Face(Vector4i indices) { + /* If the w-index is greater than -1 a QUAD face is created. */ + if (indices.w > -1) { + this.type = FaceType.QUAD; + this.vertices = new Vertex[4]; + this.vertexIndices = new int[4]; + } else { + this.type = FaceType.TRI; + this.vertices = new Vertex[3]; + this.vertexIndices = new int[3]; + } + + /* Store vertex-indices. */ + this.vertexIndices[0] = indices.x; + this.vertexIndices[1] = indices.y; + this.vertexIndices[2] = indices.z; + if (this.getType() == FaceType.QUAD) { + this.vertexIndices[3] = indices.w; + } + + /* Add vertices to face. */ + this.vertices[0] = Mesh.this.vertices.get(this.vertexIndices[0]); + this.vertices[1] = Mesh.this.vertices.get(this.vertexIndices[1]); + this.vertices[2] = Mesh.this.vertices.get(this.vertexIndices[2]); + if (this.getType() == FaceType.QUAD) { + this.vertices[3] = Mesh.this.vertices.get(this.vertexIndices[3]); + } + + /* Attach face to vertices. */ + this.vertices[0].attachToFace(this); + this.vertices[1].attachToFace(this); + this.vertices[2].attachToFace(this); + if (this.getType() == FaceType.QUAD) { + this.vertices[3].attachToFace(this); + } } /** - * Calculates and returns the Mesh's barycenter. + * Returns the list of vertices that make up this face. * - * @return Barycenter of the Mesh. + * @return Unmodifiable list of vertices. */ - @Override - public synchronized Vector3fc barycenter() { - if (this.barycenter == null) { - this.barycenter = MeshMathUtil.barycenter(this); - } - return this.barycenter; + public final List getVertices() { + return Arrays.asList(this.vertices); } /** - * Moves the Mesh in the direction of the provided vector. + * Calculates and returns the area of a face. * - * @param translation Vector describing the translation in the three directions. + * @return Area of the face. */ - @Override - public synchronized final void move(Vector3f translation) { - Matrix4f translationMatrix = new Matrix4f().translation(translation); - for (Mesh.Vertex v : this.vertices) { - v.position.mulPosition(translationMatrix); - } - if (this.barycenter != null) { - this.barycenter.mulPosition(translationMatrix); + public double area() { + if (this.type == FaceType.TRI) { + /* Extract vertices. */ + Vector3f v1 = this.vertices[0].position; + Vector3f v2 = this.vertices[1].position; + Vector3f v3 = this.vertices[2].position; + + /* Generate the edges and sort them in ascending order. */ + List edges = new ArrayList<>(); + edges.add(new Vector3f(v1).sub(v2)); + edges.add(new Vector3f(v2).sub(v3)); + edges.add(new Vector3f(v3).sub(v1)); + + edges.sort((o1, o2) -> { + float difference = o1.length() - o2.length(); + if (difference < 0) { + return -1; + } else if (difference > 0) { + return 1; + } else { + return 0; + } + }); + + float a = edges.get(2).length(); + float b = edges.get(1).length(); + float c = edges.get(0).length(); + + /* Returns the area of the triangle according to Heron's Formula. */ + double area = 0.25 * FastMath.sqrt((a + (b + c)) * (c - (a - b)) * (c + (a - b)) * (a + (b - c))); + if (Double.isNaN(area)) { + return 0.0f; + } else { + return area; } + } else { + /* Extract vertices. */ + Vector3f v1 = this.vertices[0].position; + Vector3f v2 = this.vertices[1].position; + Vector3f v3 = this.vertices[2].position; + Vector3f v4 = this.vertices[3].position; + + /* Calculates the area of the face using Bretschneider's Formula. */ + Vector3f s1 = new Vector3f(v1).sub(v2); + Vector3f s2 = new Vector3f(v2).sub(v3); + Vector3f s3 = new Vector3f(v3).sub(v4); + Vector3f s4 = new Vector3f(v4).sub(v1); + + Vector3f d1 = new Vector3f(v1).sub(v3); + Vector3f d2 = new Vector3f(v2).sub(v4); + return 0.25 * FastMath.sqrt(4 * FastMath.pow(d1.length(), 2) * FastMath.pow(d2.length(), 2) - FastMath.pow((FastMath.pow(s2.length(), 2) + FastMath.pow(s4.length(), 2) - FastMath.pow(s1.length(), 2) - FastMath.pow(s3.length(), 2)), 2)); + } } /** - * Scales the Mesh by the provided factor. This will reset the surfaceArea and bounding-box property. - * - * @param factor Factor by which the Mesh should be scaled. Values < 1.0 will cause the Mesh to shrink. + * Calculates and returns the face normal. */ - @Override - public synchronized final void scale(float factor) { - Matrix4f scaling = new Matrix4f().scale(factor); - for (Mesh.Vertex v : this.vertices) { - v.position.mulPosition(scaling); - } - - /* Reset surface-area and bounding box, which are not invariant under scaling. */ - this.surfaceArea = null; - this.boundingbox = null; + public Vector3f normal() { + Vector3f e1 = new Vector3f(this.vertices[1].position).sub(this.vertices[0].position); + Vector3f e2 = new Vector3f(this.vertices[2].position).sub(this.vertices[0].position); + return e1.cross(e2).normalize(); } /** - * Applies a transformation matrix on the Mesh by applying it to all its vertices. - * - * Important: Because transformation matrices may invalidate all derived properties - * like barycenter or surfaceArea, all this fields are reset when invoking this method. + * Calculates and returns the centroid of the face. * - * @param transformation Transformation matrix that should be applied. + * @return Centroid of the face. */ - @Override - public synchronized final void transform(Matrix4f transformation) { - for (Mesh.Vertex v : this.vertices) { - v.position.mulPosition(transformation); - } + public Vector3f centroid() { + Vector3f centroid = new Vector3f(0f, 0f, 0f); + for (Vertex vertex : this.vertices) { + centroid.add(vertex.position); + } + + if (this.type == FaceType.TRI) { + centroid.div(3.0f); + } else { + centroid.div(4.0f); + } + + return centroid; + } + } + + /** + * Enumeration used to distinguish between triangular and quadratic faces. + */ + public enum FaceType { + TRI, QUAD; + } + + /** + * List of vertices in the Mesh. + */ + private final List vertices; + + /** + * List of faces in the mesh. + */ + private final List faces; + + /** + * The position of the Mesh's barycenter. Its value is lazily calculated during invocation of the @see barycenter() method. + */ + private Vector3f barycenter; + + /** + * The surface-area of the mesh. Its value is lazily calculated during invocation of the @see surfaceArea() method. + */ + private Double surfaceArea; + + /** + * The bounding box of the mesh. Its value is lazily calculated during invocation of the @see bounds() method. + */ + private float[] boundingbox; + + /** + * Copy constructor for Mesh. + * + * @param mesh Mesh that should be copied. + */ + public Mesh(ReadableMesh mesh) { + this(mesh.numberOfFaces(), mesh.numberOfVertices()); + + for (Vertex vertex : mesh.getVertices()) { + this.addVertex(new Vector3f(vertex.position), new Vector3f(vertex.color), new Vector3f(vertex.normal)); + } - /* Reset surface-area and bounding box, which are not invariant under scaling. */ - this.surfaceArea = null; - this.boundingbox = null; - this.barycenter = null; + for (Face face : mesh.getFaces()) { + if (face.getType() == FaceType.QUAD) { + this.addFace(new Vector4i(face.vertexIndices[0], face.vertexIndices[1], face.vertexIndices[2], face.vertexIndices[3])); + } else { + this.addFace(new Vector4i(face.vertexIndices[0], face.vertexIndices[1], face.vertexIndices[2], -1)); + } + } + } + + /** + * Default constructor. + * + * @param faces Expected number of faces (not a fixed limit). + * @param vertices Expected number of vertices (not a fixed limit). + */ + public Mesh(int faces, int vertices) { + this.faces = new ArrayList<>(faces); + this.vertices = new ArrayList<>(vertices); + } + + /** + * Adds an vector defining a vertex to the Mesh. + */ + public synchronized void addVertex(Vector3f vertex) { + this.addVertex(vertex, new Vector3f(1.0f, 1.0f, 1.0f)); + } + + /** + * Adds a vector defining a vertex to the Mesh. + */ + public synchronized void addVertex(Vector3f vertex, Vector3f color) { + this.addVertex(vertex, color, new Vector3f(0.0f, 0.0f, 0.0f)); + } + + /** + * Adds an vector defining a vertex to the Mesh. + */ + public synchronized void addVertex(Vector3f vertex, Vector3f color, Vector3f normal) { + this.vertices.add(new Vertex(vertex, color, normal)); + } + + /** + * Adds a new triangular face to the Mesh. Faces index vertices and vertex normals. + * + * @param vertices Vector3i containing the indices of the vertices. + */ + @Override + public synchronized boolean addFace(Vector3i vertices) { + int limit = this.vertices.size(); + if (vertices.x < limit && vertices.y < limit && vertices.z < limit) { + Mesh.Face face = new Face(new Vector4i(vertices, -1)); + this.faces.add(face); + return true; + } else { + return false; + } + } + + /** + * Adds a new quadratic face to the Mesh. Faces index vertices and vertex normals. + * + * @param vertices Vector4i containing the indices of the vertices. + */ + @Override + public synchronized boolean addFace(Vector4i vertices) { + int limit = this.vertices.size(); + if (vertices.x < limit && vertices.y < limit && vertices.z < limit && vertices.w < limit) { + Mesh.Face face = new Face(vertices); + this.faces.add(face); + return true; + } else { + return false; + } + } + + /** + * Accessor for an individual vertex. + * + * @param vertexIndex Index of the vertex that should be returned. + * @return Vertex. + */ + @Override + public synchronized Vertex getVertex(int vertexIndex) { + return this.vertices.get(vertexIndex); + } + + /** + * Returns the list of vertices. The returned collection is unmodifiable. + * + * @return Unmodifiable list of vertices. + */ + @Override + public List getVertices() { + return Collections.unmodifiableList(this.vertices); + } + + /** + * Returns the list of vertex-normals. The returned collection is unmodifiable. + * + * @return Unmodifiable list of faces. + */ + @Override + public List getFaces() { + return Collections.unmodifiableList(faces); + } + + /** + * Returns the number of vertices in this Mesh. + * + * @return Number of vertices. + */ + @Override + public synchronized final int numberOfVertices() { + return this.vertices.size(); + } + + /** + * Returns the number of faces in this Mesh. + * + * @return Number of faces. + */ + @Override + public synchronized final int numberOfFaces() { + return this.faces.size(); + } + + /** + * Indicates, whether the mesh is an empty Mesh or not + * + * @return True if mesh is empty, false otherwise. + */ + @Override + public synchronized final boolean isEmpty() { + return this.faces.isEmpty(); + } + + /** + * Returns the total surface area of the Mesh. + * + * @return Surface area of the mesh. + */ + @Override + public synchronized final double surfaceArea() { + if (this.surfaceArea == null) { + this.surfaceArea = 0.0; + for (Face face : this.faces) { + this.surfaceArea += face.area(); + } + } + return this.surfaceArea; + } + + /** + * Calculates and returns the Mesh's bounding-box + * + * @return Bounding-box of the mesh. + */ + @Override + public synchronized float[] bounds() { + if (this.boundingbox == null) { + this.boundingbox = MeshMathUtil.bounds(this); + } + return Arrays.copyOf(this.boundingbox, 6); + } + + /** + * Calculates and returns the Mesh's barycenter. + * + * @return Barycenter of the Mesh. + */ + @Override + public synchronized Vector3fc barycenter() { + if (this.barycenter == null) { + this.barycenter = MeshMathUtil.barycenter(this); + } + return this.barycenter; + } + + /** + * Moves the Mesh in the direction of the provided vector. + * + * @param translation Vector describing the translation in the three directions. + */ + @Override + public synchronized final void move(Vector3f translation) { + Matrix4f translationMatrix = new Matrix4f().translation(translation); + for (Mesh.Vertex v : this.vertices) { + v.position.mulPosition(translationMatrix); + } + if (this.barycenter != null) { + this.barycenter.mulPosition(translationMatrix); + } + } + + /** + * Scales the Mesh by the provided factor. This will reset the surfaceArea and bounding-box property. + * + * @param factor Factor by which the Mesh should be scaled. Values < 1.0 will cause the Mesh to shrink. + */ + @Override + public synchronized final void scale(float factor) { + Matrix4f scaling = new Matrix4f().scale(factor); + for (Mesh.Vertex v : this.vertices) { + v.position.mulPosition(scaling); } - /** - * Updates the color of a vertex. - * - * @param vertexIndex Index of the vertex that should be upadated. - * @param color New color of the vertex. - */ - @Override - public synchronized void updateColor(int vertexIndex, Color color) { - Vertex vertex = this.vertices.get(vertexIndex); - vertex.color.x = color.getRed()/255.0f; - vertex.color.y = color.getBlue()/255.0f; - vertex.color.z = color.getGreen()/255.0f; + /* Reset surface-area and bounding box, which are not invariant under scaling. */ + this.surfaceArea = null; + this.boundingbox = null; + } + + /** + * Applies a transformation matrix on the Mesh by applying it to all its vertices. + * + * Important: Because transformation matrices may invalidate all derived properties + * like barycenter or surfaceArea, all this fields are reset when invoking this method. + * + * @param transformation Transformation matrix that should be applied. + */ + @Override + public synchronized final void transform(Matrix4f transformation) { + for (Mesh.Vertex v : this.vertices) { + v.position.mulPosition(transformation); } + + /* Reset surface-area and bounding box, which are not invariant under scaling. */ + this.surfaceArea = null; + this.boundingbox = null; + this.barycenter = null; + } + + /** + * Updates the color of a vertex. + * + * @param vertexIndex Index of the vertex that should be upadated. + * @param color New color of the vertex. + */ + @Override + public synchronized void updateColor(int vertexIndex, Color color) { + Vertex vertex = this.vertices.get(vertexIndex); + vertex.color.x = color.getRed() / 255.0f; + vertex.color.y = color.getBlue() / 255.0f; + vertex.color.z = color.getGreen() / 255.0f; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/ReadableMesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/ReadableMesh.java index 44cb6b045..2c5b1221b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/ReadableMesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/ReadableMesh.java @@ -1,72 +1,72 @@ package org.vitrivr.cineast.core.data.m3d; -import org.joml.Vector3fc; - import java.util.List; +import org.joml.Vector3fc; public interface ReadableMesh { - /** - * Returns the list of vertices. The returned collection is unmodifiable. - * - * @return Unmodifiable list of vertices. - */ - List getVertices(); - /** - * Accessor for an individual vertex. - * - * @param vertexIndex Index of the vertex that should be returned. - * @return Vertex. - */ - public Mesh.Vertex getVertex(int vertexIndex); + /** + * Returns the list of vertices. The returned collection is unmodifiable. + * + * @return Unmodifiable list of vertices. + */ + List getVertices(); + + /** + * Accessor for an individual vertex. + * + * @param vertexIndex Index of the vertex that should be returned. + * @return Vertex. + */ + public Mesh.Vertex getVertex(int vertexIndex); - /** - * Returns the list of vertex-normals. The returned collection is unmodifiable. - * - * @return Unmodifiable list of faces. - */ - List getFaces(); + /** + * Returns the list of vertex-normals. The returned collection is unmodifiable. + * + * @return Unmodifiable list of faces. + */ + List getFaces(); - /** - * Returns the number of vertices in this Mesh. - * - * @return Number of vertices. - */ - int numberOfVertices(); + /** + * Returns the number of vertices in this Mesh. + * + * @return Number of vertices. + */ + int numberOfVertices(); - /** - * Returns the number of faces in this Mesh. - * - * @return Number of faces. - */ - int numberOfFaces(); + /** + * Returns the number of faces in this Mesh. + * + * @return Number of faces. + */ + int numberOfFaces(); - /** - * Returns the total surface area of the Mesh. - * - * @return Surface area of the mesh. - */ - double surfaceArea(); + /** + * Returns the total surface area of the Mesh. + * + * @return Surface area of the mesh. + */ + double surfaceArea(); - /** - * Calculates and returns the Mesh's bounding-box - * - * @return Bounding-box of the mesh. - */ - float[] bounds(); + /** + * Calculates and returns the Mesh's bounding-box + * + * @return Bounding-box of the mesh. + */ + float[] bounds(); - /** - * Calculates and returns the Mesh's barycenter. - * - * @return Barycenter of the Mesh. - */ - Vector3fc barycenter(); + /** + * Calculates and returns the Mesh's barycenter. + * + * @return Barycenter of the Mesh. + */ + Vector3fc barycenter(); - /** - * Indicates, whether the mesh is an empty Mesh or not - * - * @return True if mesh is empty, false otherwise. - */ - boolean isEmpty(); + /** + * Indicates, whether the mesh is an empty Mesh or not + * + * @return True if mesh is empty, false otherwise. + */ + boolean isEmpty(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java index 9fd524aa1..3d75dccd9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/VoxelGrid.java @@ -5,269 +5,282 @@ import org.joml.Vector3i; /** - * This class represents a Voxel grid, i.e. a 3-dimensional grid of 3D pixels (called Voxels). Every voxel - * can either be visible or invisible. - * + * This class represents a Voxel grid, i.e. a 3-dimensional grid of 3D pixels (called Voxels). Every voxel can either be visible or invisible. */ public class VoxelGrid { - /** The default, empty VoxelGrid. */ - public static final VoxelGrid EMPTY = new VoxelGrid(5,5,5,0.1f); - /** Represents a single Voxel which can either can be visible or invisible. */ - public enum Voxel { - VISIBLE, INVISIBLE - } + /** + * The default, empty VoxelGrid. + */ + public static final VoxelGrid EMPTY = new VoxelGrid(5, 5, 5, 0.1f); - /** Determines the size of a single voxel. */ - private float resolution; + /** + * Represents a single Voxel which can either can be visible or invisible. + */ + public enum Voxel { + VISIBLE, INVISIBLE + } - /** The size of the voxel grid in X direction. */ - private final int sizeX; + /** + * Determines the size of a single voxel. + */ + private float resolution; - /** The size of the voxel grid in X direction. */ - private final int sizeY; + /** + * The size of the voxel grid in X direction. + */ + private final int sizeX; - /** The size of the voxel grid in X direction. */ - private final int sizeZ; + /** + * The size of the voxel grid in X direction. + */ + private final int sizeY; - /** The total length ot the voxel grid (i.e. the number of voxels in the grid). */ - private final int length; + /** + * The size of the voxel grid in X direction. + */ + private final int sizeZ; - /** Number of visible voxels in the grid. */ - private int visible = 0; + /** + * The total length ot the voxel grid (i.e. the number of voxels in the grid). + */ + private final int length; - /** Number of invisible voxels in the grid. */ - private int invisible = 0; + /** + * Number of visible voxels in the grid. + */ + private int visible = 0; - /** Array holding the actual voxels. */ - private final Voxel[][][] voxelGrid; + /** + * Number of invisible voxels in the grid. + */ + private int invisible = 0; - /** Defines the center of the voxel-grid (in the world coordinate system). It corresponds to the - * center of the voxel at (sizeX/2, sizeY/2, sizeZ/2). - * - * - *

    Important:

    Transformation into world coordinates are based on this center! - */ - private final Vector3f center = new Vector3f(0f,0f,0f); + /** + * Array holding the actual voxels. + */ + private final Voxel[][][] voxelGrid; - /** - * Default constructor: Initializes a new, fully active Voxel-Grid - * - * @param sizeX X-size of the new voxel grid. - * @param sizeY Y-size of the new voxel grid. - * @param sizeZ Z-size of the new voxel grid. - * @param resolution Resolution of the grid, i.e. size of a single voxel. - */ - public VoxelGrid(int sizeX, int sizeY, int sizeZ, float resolution) { - this(sizeX, sizeY, sizeZ, resolution, true); - } + /** + * Defines the center of the voxel-grid (in the world coordinate system). It corresponds to the center of the voxel at (sizeX/2, sizeY/2, sizeZ/2). + * + * + *

    Important:

    Transformation into world coordinates are based on this center! + */ + private final Vector3f center = new Vector3f(0f, 0f, 0f); + + /** + * Default constructor: Initializes a new, fully active Voxel-Grid + * + * @param sizeX X-size of the new voxel grid. + * @param sizeY Y-size of the new voxel grid. + * @param sizeZ Z-size of the new voxel grid. + * @param resolution Resolution of the grid, i.e. size of a single voxel. + */ + public VoxelGrid(int sizeX, int sizeY, int sizeZ, float resolution) { + this(sizeX, sizeY, sizeZ, resolution, true); + } - /** - * Default constructor: Initializes a new Voxel-Grid. - * - * @param sizeX X-size of the new voxel grid. - * @param sizeY Y-size of the new voxel grid. - * @param sizeZ Z-size of the new voxel grid. - * @param resolution Resolution of the grid, i.e. size of a single voxel. - * @param active Indicates whether the grid should be initialized with active or inactive Voxels. - */ - public VoxelGrid(int sizeX, int sizeY, int sizeZ, float resolution, boolean active) { - this.sizeX = sizeX; - this.sizeY = sizeY; - this.sizeZ = sizeZ; - this.resolution = resolution; - this.length = sizeX * sizeY * sizeZ; - this.voxelGrid = new Voxel[sizeX][sizeY][sizeZ]; - for (short i = 0;i 0; - } + /** + * Returns true if VoxelGrid is visible (i.e. there is at least one visible Voxel) and false otherwise. + */ + public final boolean isVisible() { + return this.visible > 0; + } - /** - * Returns the resolution of the Voxel grid, i.e. the size of a single - * Voxel. - * - * @return Size of a single voxel. - */ - public final float getResolution() { - return resolution; - } + /** + * Returns the resolution of the Voxel grid, i.e. the size of a single Voxel. + * + * @return Size of a single voxel. + */ + public final float getResolution() { + return resolution; + } - /** - * Transforms world-coordinates (e.g. position of a vertex in space) into the corresponding voxel coordinates, - * i.e. the index of the voxel in the grid. - * - *

    Important:

    The indices returned by this method are not necessarily within the bounds - * of the grid. - * - * @param coordinate Coordinates to be transformed. - * @return coordinate Voxel indices. - */ - public final Vector3i coordinateToVoxel(Vector3fc coordinate) { - Vector3f gridCoordinates = (new Vector3f(coordinate)).add(this.center).div(this.resolution); - return new Vector3i((int)Math.ceil(gridCoordinates.x + this.sizeX/2), (int)Math.ceil(gridCoordinates.y + this.sizeY/2), (int)Math.ceil(gridCoordinates.z + this.sizeZ/2)); - } + /** + * Transforms world-coordinates (e.g. position of a vertex in space) into the corresponding voxel coordinates, i.e. the index of the voxel in the grid. + * + *

    Important:

    The indices returned by this method are not necessarily within the bounds + * of the grid. + * + * @param coordinate Coordinates to be transformed. + * @return coordinate Voxel indices. + */ + public final Vector3i coordinateToVoxel(Vector3fc coordinate) { + Vector3f gridCoordinates = (new Vector3f(coordinate)).add(this.center).div(this.resolution); + return new Vector3i((int) Math.ceil(gridCoordinates.x + this.sizeX / 2), (int) Math.ceil(gridCoordinates.y + this.sizeY / 2), (int) Math.ceil(gridCoordinates.z + this.sizeZ / 2)); + } - /** - * Returns the Voxel at the specified position. - * - * @param x x position of the Voxel. - * @param y y position of the Voxel. - * @param z z position of the Voxel. - * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. - */ - public final Voxel get(int x, int y, int z) { - return this.voxelGrid[x][y][z]; - } + /** + * Returns the Voxel at the specified position. + * + * @param x x position of the Voxel. + * @param y y position of the Voxel. + * @param z z position of the Voxel. + * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. + */ + public final Voxel get(int x, int y, int z) { + return this.voxelGrid[x][y][z]; + } - /** - * Returns true, if the Voxel at the specified position is visible and false otherwise. - * - * @param x x position of the Voxel. - * @param y y position of the Voxel. - * @param z z position of the Voxel. - * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. - */ - public final boolean isVisible(int x, int y, int z) { - return this.voxelGrid[x][y][z] == Voxel.VISIBLE; - } + /** + * Returns true, if the Voxel at the specified position is visible and false otherwise. + * + * @param x x position of the Voxel. + * @param y y position of the Voxel. + * @param z z position of the Voxel. + * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. + */ + public final boolean isVisible(int x, int y, int z) { + return this.voxelGrid[x][y][z] == Voxel.VISIBLE; + } - /** - * Calculates and returns the center of the Voxel in a 3D coordinate system using the grids - * resolution property. - * - * @param x x position of the Voxel. - * @param y y position of the Voxel. - * @param z z position of the Voxel. - * @return Vector3f containing the center of the voxel. - * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. - */ - public Vector3f getVoxelCenter(int x, int y, int z) { - return new Vector3f((x-VoxelGrid.this.sizeX/2) * VoxelGrid.this.resolution + this.center.x, (y-VoxelGrid.this.sizeY/2)*VoxelGrid.this.resolution + this.center.y, (z-VoxelGrid.this.sizeZ/2)*VoxelGrid.this.resolution + this.center.z); - } + /** + * Calculates and returns the center of the Voxel in a 3D coordinate system using the grids resolution property. + * + * @param x x position of the Voxel. + * @param y y position of the Voxel. + * @param z z position of the Voxel. + * @return Vector3f containing the center of the voxel. + * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. + */ + public Vector3f getVoxelCenter(int x, int y, int z) { + return new Vector3f((x - VoxelGrid.this.sizeX / 2) * VoxelGrid.this.resolution + this.center.x, (y - VoxelGrid.this.sizeY / 2) * VoxelGrid.this.resolution + this.center.y, (z - VoxelGrid.this.sizeZ / 2) * VoxelGrid.this.resolution + this.center.z); + } - /** - * Toggles the Voxel at the specified position. - * - * @param visible If true, the new Voxel position will become visible. - * @param x x position of the Voxel. - * @param y y position of the Voxel. - * @param z z position of the Voxel. - * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. - */ - public final void toggleVoxel(boolean visible, int x, int y, int z) { - if (visible && this.voxelGrid[x][y][z] == Voxel.INVISIBLE) { - this.voxelGrid[x][y][z] = Voxel.VISIBLE; - this.invisible -= 1; - this.visible += 1; - } else if (!visible && this.voxelGrid[x][y][z] == Voxel.VISIBLE) { - this.voxelGrid[x][y][z] = Voxel.INVISIBLE; - this.invisible += 1; - this.visible -= 1; - } + /** + * Toggles the Voxel at the specified position. + * + * @param visible If true, the new Voxel position will become visible. + * @param x x position of the Voxel. + * @param y y position of the Voxel. + * @param z z position of the Voxel. + * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. + */ + public final void toggleVoxel(boolean visible, int x, int y, int z) { + if (visible && this.voxelGrid[x][y][z] == Voxel.INVISIBLE) { + this.voxelGrid[x][y][z] = Voxel.VISIBLE; + this.invisible -= 1; + this.visible += 1; + } else if (!visible && this.voxelGrid[x][y][z] == Voxel.VISIBLE) { + this.voxelGrid[x][y][z] = Voxel.INVISIBLE; + this.invisible += 1; + this.visible -= 1; } + } - /** - * Converts the VoxelGrid into a string that can be read by Matlab (e.g. for 3D scatter plots). - * The array contains the coordinates of all visible voxels. - * - * @return String - */ - public String toMatlabArray() { - StringBuilder buffer = new StringBuilder(); - buffer.append("["); - for (int x=0;x + * The resulting VoxelGrid approximates the surface of the Mesh. The volumetric interior of the grid will not be filled! * *

    Important:

    The resolution of the Voxelizer and the size of the Mesh determine the accuracy of the Voxelization - * process. For instance, if the resolution is chosen such that one Voxel has the size of the whole mesh, the Voxelization - * will not yield any meaningful result. - * - * [1] Huang, J. H. J., Yagel, R. Y. R., Filippov, V. F. V., & Kurzion, Y. K. Y. (1998). - * An accurate method for voxelizing polygon meshes. IEEE Symposium on Volume Visualization (Cat. No.989EX300), 119–126,. http://doi.org/10.1109/SVV.1998.729593 - * + * process. For instance, if the resolution is chosen such that one Voxel has the size of the whole mesh, the Voxelization will not yield any meaningful result. + *

    + * [1] Huang, J. H. J., Yagel, R. Y. R., Filippov, V. F. V., & Kurzion, Y. K. Y. (1998). An accurate method for voxelizing polygon meshes. IEEE Symposium on Volume Visualization (Cat. No.989EX300), 119–126,. http://doi.org/10.1109/SVV.1998.729593 */ public class Voxelizer { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Resolution, i.e. size of a single voxel. */ - private final float resolution; - - /** Half of the resolution. Pre-calculated for convenience. */ - private final float rc; - - /** Half the resolution squared. Pre-calculated for convenience. */ - private final float rcsq; - - /** - * Default constructor. Defines the resolution of the Voxelizer. - * - * @param resolution Resolution of the Voxelizer (i.e. size of a single Voxel in the grid). This is an important - * parameter, as it determines the quality of the Voxelization process! - */ - public Voxelizer(float resolution) { - this.resolution = resolution; - this.rc = resolution/2; - this.rcsq = (float)Math.pow(this.rc,2); - } - - /** - * Voxelizes the provided mesh returning a VoxelGrid with the specified resolution. The size of the VoxelGrid - * will be chose so as to fit the size of the mesh. - * - * @param mesh Mesh that should be voxelized. - * @return VoxelGrid representation of the mesh. - */ - public VoxelGrid voxelize(ReadableMesh mesh) { - /* Calculate bounding box of mesh. */ - float[] boundingBox = mesh.bounds(); - short sizeX = (short)(Math.abs(Math.ceil(((boundingBox[0]-boundingBox[1])/this.resolution))) + 1); - short sizeY = (short)(Math.abs(Math.ceil(((boundingBox[2]-boundingBox[3])/this.resolution))) + 1); - short sizeZ = (short)(Math.abs(Math.ceil(((boundingBox[4]-boundingBox[5])/this.resolution))) + 1); - - /* Return the voxelized mesh. */ - return this.voxelize(mesh, sizeX, sizeY, sizeZ); - } - /** - * Voxelizes the provided mesh returning a new VoxelGrid with the specified resolution. The size of - * the VoxelGrid will be fixed - * - * @param mesh Mesh that should be voxelized. - * @param sizeX Number of Voxels in X direction. - * @param sizeY Number of Voxels in Y direction. - * @param sizeZ Number of Voxels in Z direction. - * @return VoxelGrid representation of the mesh. - */ - public VoxelGrid voxelize(ReadableMesh mesh, int sizeX, int sizeY, int sizeZ) { - /* Initializes a new voxel-grid. */ - VoxelGrid grid = new VoxelGrid(sizeX, sizeY, sizeZ, this.resolution, false); - - /* Return the voxelized mesh. */ - return this.voxelize(mesh, grid); - } - - /** - * Voxelizes the provided mesh into the provided VoxelGrid - * - * @param mesh Mesh that should be voxelized. - * @param grid VoxelGrid to use for voxelization. - * @return VoxelGrid representation of the mesh. - */ - public VoxelGrid voxelize(ReadableMesh mesh, VoxelGrid grid) { - - long start = System.currentTimeMillis(); - - /* Process the faces and perform all the relevant tests described in [1]. */ - for (Mesh.Face face : mesh.getFaces()) { - List vertices = face.getVertices(); - List> enclosings = Voxelizer.this.enclosingGrid(vertices, grid); - for (Pair enclosing : enclosings) { - /* Perform vertex-tests. */ - if (Voxelizer.this.vertextTest(vertices.get(0), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - if (Voxelizer.this.vertextTest(vertices.get(1), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - if (Voxelizer.this.vertextTest(vertices.get(2), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - /* Perform edge-tests. */ - if (Voxelizer.this.edgeTest(vertices.get(0), vertices.get(1), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - if (Voxelizer.this.edgeTest(vertices.get(1), vertices.get(2), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - if (Voxelizer.this.edgeTest(vertices.get(2), vertices.get(0), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - continue; - } - - /* Perform plane-tests. */ - if (Voxelizer.this.planeTest(vertices.get(0), vertices.get(1), vertices.get(2), enclosing)) { - grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); - } - } + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Resolution, i.e. size of a single voxel. + */ + private final float resolution; + + /** + * Half of the resolution. Pre-calculated for convenience. + */ + private final float rc; + + /** + * Half the resolution squared. Pre-calculated for convenience. + */ + private final float rcsq; + + /** + * Default constructor. Defines the resolution of the Voxelizer. + * + * @param resolution Resolution of the Voxelizer (i.e. size of a single Voxel in the grid). This is an important parameter, as it determines the quality of the Voxelization process! + */ + public Voxelizer(float resolution) { + this.resolution = resolution; + this.rc = resolution / 2; + this.rcsq = (float) Math.pow(this.rc, 2); + } + + /** + * Voxelizes the provided mesh returning a VoxelGrid with the specified resolution. The size of the VoxelGrid will be chose so as to fit the size of the mesh. + * + * @param mesh Mesh that should be voxelized. + * @return VoxelGrid representation of the mesh. + */ + public VoxelGrid voxelize(ReadableMesh mesh) { + /* Calculate bounding box of mesh. */ + float[] boundingBox = mesh.bounds(); + short sizeX = (short) (Math.abs(Math.ceil(((boundingBox[0] - boundingBox[1]) / this.resolution))) + 1); + short sizeY = (short) (Math.abs(Math.ceil(((boundingBox[2] - boundingBox[3]) / this.resolution))) + 1); + short sizeZ = (short) (Math.abs(Math.ceil(((boundingBox[4] - boundingBox[5]) / this.resolution))) + 1); + + /* Return the voxelized mesh. */ + return this.voxelize(mesh, sizeX, sizeY, sizeZ); + } + + /** + * Voxelizes the provided mesh returning a new VoxelGrid with the specified resolution. The size of the VoxelGrid will be fixed + * + * @param mesh Mesh that should be voxelized. + * @param sizeX Number of Voxels in X direction. + * @param sizeY Number of Voxels in Y direction. + * @param sizeZ Number of Voxels in Z direction. + * @return VoxelGrid representation of the mesh. + */ + public VoxelGrid voxelize(ReadableMesh mesh, int sizeX, int sizeY, int sizeZ) { + /* Initializes a new voxel-grid. */ + VoxelGrid grid = new VoxelGrid(sizeX, sizeY, sizeZ, this.resolution, false); + + /* Return the voxelized mesh. */ + return this.voxelize(mesh, grid); + } + + /** + * Voxelizes the provided mesh into the provided VoxelGrid + * + * @param mesh Mesh that should be voxelized. + * @param grid VoxelGrid to use for voxelization. + * @return VoxelGrid representation of the mesh. + */ + public VoxelGrid voxelize(ReadableMesh mesh, VoxelGrid grid) { + + long start = System.currentTimeMillis(); + + /* Process the faces and perform all the relevant tests described in [1]. */ + for (Mesh.Face face : mesh.getFaces()) { + List vertices = face.getVertices(); + List> enclosings = Voxelizer.this.enclosingGrid(vertices, grid); + for (Pair enclosing : enclosings) { + /* Perform vertex-tests. */ + if (Voxelizer.this.vertextTest(vertices.get(0), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; + } + if (Voxelizer.this.vertextTest(vertices.get(1), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; + } + if (Voxelizer.this.vertextTest(vertices.get(2), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; + } + /* Perform edge-tests. */ + if (Voxelizer.this.edgeTest(vertices.get(0), vertices.get(1), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; + } + if (Voxelizer.this.edgeTest(vertices.get(1), vertices.get(2), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; + } + if (Voxelizer.this.edgeTest(vertices.get(2), vertices.get(0), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); + continue; } - long stop = System.currentTimeMillis(); - LOGGER.log(Level.DEBUG, String.format("Voxelization of mesh completed in %d ms (Size: %d x %d x %d).", (stop - start), grid.getSizeX(), grid.getSizeY(), grid.getSizeZ())); - - return grid; - } - - /** - * Performs the vertex-test described in [1]. Checks if the provided voxel's center is within the area - * of circle with radius L/2 around the vertex (L being the size of a voxel). - * - * @param vertex Vertex to be tested. - * @param voxel Voxel to be tested. - * @return true if the voxel's center is within the circle, false otherwise. - */ - private boolean vertextTest(Mesh.Vertex vertex, Pair voxel) { - return vertex.getPosition().distanceSquared(voxel.second) > this.rcsq; - } - - /** - * Performs the edge-test described in [1]. Checks if the provided voxel's center is enclosed in the cylinder - * around the line between vertex a and vertex b. - * - * @param a First vertex that constitutes the line used to draw a cylinder around. - * @param b Second vertex that constitutes the line used to draw a cylinder around. - * @param voxel Voxel to be tested. - * @return true if voxel's center is contained in cylinder, false otherwise. - */ - private boolean edgeTest(Mesh.Vertex a, Mesh.Vertex b, Pair voxel) { - Vector3f line = (new Vector3f(b.getPosition())).sub(a.getPosition()); - Vector3f pd = voxel.second.sub(a.getPosition()); - - /* Calculate distance between a and b (Edge). */ - float lsq = a.getPosition().distanceSquared(b.getPosition()); - float dot = line.dot(pd); - - if (dot < 0.0f || dot > lsq) { - return false; - } else { - float dsq = pd.lengthSquared() - ((float)Math.pow(dot,2))/lsq; - return dsq > rc; + /* Perform plane-tests. */ + if (Voxelizer.this.planeTest(vertices.get(0), vertices.get(1), vertices.get(2), enclosing)) { + grid.toggleVoxel(true, enclosing.first.x, enclosing.first.y, enclosing.first.z); } + } } - /** - * Performs a simplified version of the plane-test described in [1]. Checks if the provided voxel's center is - * enclosed in the space spanned by two planes parallel to the face. - * - * The original version of the test performs three addition checks with planes that go through the edges. These - * tests are ommited because we only work on a reduced set of voxels that directly enclose the vertices in question. - * - * @param a First vertex that spans the face. - * @param b Second vertex that spans the face. - * @param c Third vertex that spans the face. - * @param voxel Voxel to be tested. - * @return true if voxel's center is contained in the area, false otherwise. - */ - private boolean planeTest(Mesh.Vertex a, Mesh.Vertex b, Mesh.Vertex c, Pair voxel) { - /* Retrieve center and corner of voxel. */ - Vector3f vcenter = voxel.second; - Vector3f vcorner = (new Vector3f(this.rc, this.rc, this.rc)).add(vcenter); - - /* Calculate the vectors spanning the plane of the facepolyon and its plane-normal. */ - Vector3f ab = (new Vector3f(b.getPosition())).sub(a.getPosition()); - Vector3f ac = (new Vector3f(c.getPosition())).sub(a.getPosition()); - Vector3f planenorm = (new Vector3f(ab)).cross(ac); - - /* Calculate the distance t for enclosing planes. */ - float t = (float)(this.rc* MathConstants.SQRT3*vcorner.angleCos(vcenter)); - - /* Derive new displaced plane normals. */ - Vector3f planenorm_plus = new Vector3f(planenorm.x + t, planenorm.y + t, planenorm.z + t); - Vector3f planenorm_minus = new Vector3f(planenorm.x - t, planenorm.y - t, planenorm.z - t); - - /* Check if the center is under the planenorm_plus and above the planenorm_minus. */ - return planenorm_plus.dot(vcenter) < 0 && planenorm_minus.dot(vcenter) > 0; + long stop = System.currentTimeMillis(); + LOGGER.log(Level.DEBUG, String.format("Voxelization of mesh completed in %d ms (Size: %d x %d x %d).", (stop - start), grid.getSizeX(), grid.getSizeY(), grid.getSizeZ())); + + return grid; + } + + /** + * Performs the vertex-test described in [1]. Checks if the provided voxel's center is within the area of circle with radius L/2 around the vertex (L being the size of a voxel). + * + * @param vertex Vertex to be tested. + * @param voxel Voxel to be tested. + * @return true if the voxel's center is within the circle, false otherwise. + */ + private boolean vertextTest(Mesh.Vertex vertex, Pair voxel) { + return vertex.getPosition().distanceSquared(voxel.second) > this.rcsq; + } + + /** + * Performs the edge-test described in [1]. Checks if the provided voxel's center is enclosed in the cylinder around the line between vertex a and vertex b. + * + * @param a First vertex that constitutes the line used to draw a cylinder around. + * @param b Second vertex that constitutes the line used to draw a cylinder around. + * @param voxel Voxel to be tested. + * @return true if voxel's center is contained in cylinder, false otherwise. + */ + private boolean edgeTest(Mesh.Vertex a, Mesh.Vertex b, Pair voxel) { + Vector3f line = (new Vector3f(b.getPosition())).sub(a.getPosition()); + Vector3f pd = voxel.second.sub(a.getPosition()); + + /* Calculate distance between a and b (Edge). */ + float lsq = a.getPosition().distanceSquared(b.getPosition()); + float dot = line.dot(pd); + + if (dot < 0.0f || dot > lsq) { + return false; + } else { + float dsq = pd.lengthSquared() - ((float) Math.pow(dot, 2)) / lsq; + return dsq > rc; + } + } + + /** + * Performs a simplified version of the plane-test described in [1]. Checks if the provided voxel's center is enclosed in the space spanned by two planes parallel to the face. + *

    + * The original version of the test performs three addition checks with planes that go through the edges. These tests are ommited because we only work on a reduced set of voxels that directly enclose the vertices in question. + * + * @param a First vertex that spans the face. + * @param b Second vertex that spans the face. + * @param c Third vertex that spans the face. + * @param voxel Voxel to be tested. + * @return true if voxel's center is contained in the area, false otherwise. + */ + private boolean planeTest(Mesh.Vertex a, Mesh.Vertex b, Mesh.Vertex c, Pair voxel) { + /* Retrieve center and corner of voxel. */ + Vector3f vcenter = voxel.second; + Vector3f vcorner = (new Vector3f(this.rc, this.rc, this.rc)).add(vcenter); + + /* Calculate the vectors spanning the plane of the facepolyon and its plane-normal. */ + Vector3f ab = (new Vector3f(b.getPosition())).sub(a.getPosition()); + Vector3f ac = (new Vector3f(c.getPosition())).sub(a.getPosition()); + Vector3f planenorm = (new Vector3f(ab)).cross(ac); + + /* Calculate the distance t for enclosing planes. */ + float t = (float) (this.rc * MathConstants.SQRT3 * vcorner.angleCos(vcenter)); + + /* Derive new displaced plane normals. */ + Vector3f planenorm_plus = new Vector3f(planenorm.x + t, planenorm.y + t, planenorm.z + t); + Vector3f planenorm_minus = new Vector3f(planenorm.x - t, planenorm.y - t, planenorm.z - t); + + /* Check if the center is under the planenorm_plus and above the planenorm_minus. */ + return planenorm_plus.dot(vcenter) < 0 && planenorm_minus.dot(vcenter) > 0; + } + + /** + * Calculates and returns the enclosing grid, i.e. a list of Voxels from the grid that enclose the list of provided vertices. + * + * @param vertices The Vertices for which an enclosing grid needs to be found. + * @param grid VoxelGrid to select voxels from. + * @return List of voxels that confine the provided vertices. + */ + private List> enclosingGrid(List vertices, VoxelGrid grid) { + /* Calculate bounding box for provided vertices. */ + ArrayList positions = new ArrayList<>(vertices.size()); + for (Mesh.Vertex vertex : vertices) { + positions.add(vertex.getPosition()); } - /** - * Calculates and returns the enclosing grid, i.e. a list of Voxels from the grid that enclose the list - * of provided vertices. - * - * @param vertices The Vertices for which an enclosing grid needs to be found. - * @param grid VoxelGrid to select voxels from. - * @return List of voxels that confine the provided vertices. - */ - private List> enclosingGrid(List vertices, VoxelGrid grid) { - /* Calculate bounding box for provided vertices. */ - ArrayList positions = new ArrayList<>(vertices.size()); - for (Mesh.Vertex vertex : vertices) { - positions.add(vertex.getPosition()); - } - - float bounds[] = MeshMathUtil.bounds(positions); + float bounds[] = MeshMathUtil.bounds(positions); - /* Derive max and min voxel-indices from bounding-boxes. */ - Vector3i max = grid.coordinateToVoxel(new Vector3f(bounds[0], bounds[2], bounds[4])); - Vector3i min = grid.coordinateToVoxel(new Vector3f(bounds[1], bounds[3], bounds[5])); + /* Derive max and min voxel-indices from bounding-boxes. */ + Vector3i max = grid.coordinateToVoxel(new Vector3f(bounds[0], bounds[2], bounds[4])); + Vector3i min = grid.coordinateToVoxel(new Vector3f(bounds[1], bounds[3], bounds[5])); - /* Initialize an empty ArrayList for the Voxel-Elements. */ - List> enclosing = new ArrayList<>((max.x-min.x) * (max.y-min.y) * (max.z-min.z)); - for (int i = min.x; i <= max.x; i++) { - for (int j = min.y;j<= max.y; j++) { - for (int k = min.z; k <= max.z; k++) { - enclosing.add(new Pair<>(new Vector3i(i,j,k), grid.getVoxelCenter(i,j,k))); - } - } + /* Initialize an empty ArrayList for the Voxel-Elements. */ + List> enclosing = new ArrayList<>((max.x - min.x) * (max.y - min.y) * (max.z - min.z)); + for (int i = min.x; i <= max.x; i++) { + for (int j = min.y; j <= max.y; j++) { + for (int k = min.z; k <= max.z; k++) { + enclosing.add(new Pair<>(new Vector3i(i, j, k), grid.getVoxelCenter(i, j, k))); } - - /* Return list of enclosing voxels. */ - return enclosing; + } } + + /* Return list of enclosing voxels. */ + return enclosing; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/WritableMesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/WritableMesh.java index 66f4e62b3..9f7f2e038 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/WritableMesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/WritableMesh.java @@ -1,58 +1,56 @@ package org.vitrivr.cineast.core.data.m3d; +import java.awt.Color; import org.joml.Matrix4f; import org.joml.Vector3f; import org.joml.Vector3i; import org.joml.Vector4i; -import java.awt.*; - public interface WritableMesh extends ReadableMesh { - /** - * Adds a new triangular face to the Mesh. Faces index vertices and - * vertex normals. - * - * @param vertices Vector3i containing the indices of the vertices. - * @return true if face could be added and false otherwise (e.g. if indices point to non-existing vertex) - */ - boolean addFace(Vector3i vertices); - - /** - * Adds a new quadratic face to the Mesh. Faces index vertices and - * vertex normals. - * - * @param vertices Vector4i containing the indices of the vertices. - * @return true if face could be added and false otherwise (e.g. if indices point to non-existing vertex) - */ - boolean addFace(Vector4i vertices); - - /** - * Scales the Mesh by the provided factor. This will reset the surfaceArea and bounding-box property. - * - * @param factor Factor by which the Mesh should be scaled. Values < 1.0 will cause the Mesh to shrink. - */ - void scale(float factor); - - /** - * Moves the Mesh in the direction of the provided vector. - * - * @param translation Vector describing the translation in the three directions. - */ - void move(Vector3f translation); - - /** - * Applies a transformation matrix on the Mesh by applying it to all its vertices. - * - * @param transformation Transformation matrix that should be applied. - */ - void transform(Matrix4f transformation); - - /** - * Updates the color of a vertex. - * - * @param vertexIndex Index of the vertex that should be upadated. - * @param color New color of the vertex. - */ - void updateColor(int vertexIndex, Color color); + + /** + * Adds a new triangular face to the Mesh. Faces index vertices and vertex normals. + * + * @param vertices Vector3i containing the indices of the vertices. + * @return true if face could be added and false otherwise (e.g. if indices point to non-existing vertex) + */ + boolean addFace(Vector3i vertices); + + /** + * Adds a new quadratic face to the Mesh. Faces index vertices and vertex normals. + * + * @param vertices Vector4i containing the indices of the vertices. + * @return true if face could be added and false otherwise (e.g. if indices point to non-existing vertex) + */ + boolean addFace(Vector4i vertices); + + /** + * Scales the Mesh by the provided factor. This will reset the surfaceArea and bounding-box property. + * + * @param factor Factor by which the Mesh should be scaled. Values < 1.0 will cause the Mesh to shrink. + */ + void scale(float factor); + + /** + * Moves the Mesh in the direction of the provided vector. + * + * @param translation Vector describing the translation in the three directions. + */ + void move(Vector3f translation); + + /** + * Applies a transformation matrix on the Mesh by applying it to all its vertices. + * + * @param transformation Transformation matrix that should be applied. + */ + void transform(Matrix4f transformation); + + /** + * Updates the color of a vertex. + * + * @param vertexIndex Index of the vertex that should be upadated. + * @param color New color of the vertex. + */ + void updateColor(int vertexIndex, Color color); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioFrameProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioFrameProvider.java index 0d51c96aa..a192140e5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioFrameProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioFrameProvider.java @@ -1,134 +1,126 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.frames.AudioFrame; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.data.frames.AudioFrame; /** - * * This interface should be implemented by segments that provides access to frames-data in the form of {@link AudioFrame}. - * - * Currently, one AudioFrame holds an arbitrary number of samples and channels (as returned by the respective decoder). An - * audio-segment groups multiple such frames and the methods defined in this interface facilitate easy access to the - * underlying data. - * + *

    + * Currently, one AudioFrame holds an arbitrary number of samples and channels (as returned by the respective decoder). An audio-segment groups multiple such frames and the methods defined in this interface facilitate easy access to the underlying data. */ public interface AudioFrameProvider { - /** - * Returns a list of audio-frames contained in the AudioSegment. The - * default implementation returns a list containing one, empty frame. - * - * @return List auf audio-frames in the audio-segment. - */ - default List getAudioFrames() { - final ArrayList list = new ArrayList<>(); - list.add(AudioFrame.EMPTY_FRAME); - return list; - } - /** - * Returns the raw samples in the specified channel as short array. - * - * @param channel The channel for which to return the frames-data (zero-based index). - * @return short array containing the samples. - */ - default short[] getSamplesAsShort(int channel) { - short[] samples = new short[this.getNumberOfSamples()]; - int idx = 0; - for (AudioFrame frame : this.getAudioFrames()) { - for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { - samples[idx] = frame.getSampleAsShort(sample, channel); - } - } - return samples; - } + /** + * Returns a list of audio-frames contained in the AudioSegment. The default implementation returns a list containing one, empty frame. + * + * @return List auf audio-frames in the audio-segment. + */ + default List getAudioFrames() { + final ArrayList list = new ArrayList<>(); + list.add(AudioFrame.EMPTY_FRAME); + return list; + } - /** - * Returns the raw samples in the specified channel as double array. - * - * @param channel The channel for which to return the frames-data (zero-based index). - * @return double array containing the samples. - */ - default double[] getSamplesAsDouble(int channel) { - double[] samples = new double[this.getNumberOfSamples()]; - int idx = 0; - for (AudioFrame frame : this.getAudioFrames()) { - for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { - samples[idx] = frame.getSampleAsDouble(sample, channel); - } - } - return samples; + /** + * Returns the raw samples in the specified channel as short array. + * + * @param channel The channel for which to return the frames-data (zero-based index). + * @return short array containing the samples. + */ + default short[] getSamplesAsShort(int channel) { + short[] samples = new short[this.getNumberOfSamples()]; + int idx = 0; + for (AudioFrame frame : this.getAudioFrames()) { + for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { + samples[idx] = frame.getSampleAsShort(sample, channel); + } } + return samples; + } - /** - * Returns the mean samples across all channels as short array. - * - * @return short array containing the mean sample values. - */ - default short[] getMeanSamplesAsShort() { - short[] samples = new short[this.getNumberOfSamples()]; - int idx = 0; - for (AudioFrame frame : this.getAudioFrames()) { - for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { - samples[idx] = frame.getMeanSampleAsShort(sample); - } - } - return samples; + /** + * Returns the raw samples in the specified channel as double array. + * + * @param channel The channel for which to return the frames-data (zero-based index). + * @return double array containing the samples. + */ + default double[] getSamplesAsDouble(int channel) { + double[] samples = new double[this.getNumberOfSamples()]; + int idx = 0; + for (AudioFrame frame : this.getAudioFrames()) { + for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { + samples[idx] = frame.getSampleAsDouble(sample, channel); + } } + return samples; + } - /** - * Returns the mean samples across all channels as double array. - * - * @return double array containing the mean sample values. - */ - default double[] getMeanSamplesAsDouble() { - double[] samples = new double[this.getNumberOfSamples()]; - int idx = 0; - for (AudioFrame frame : this.getAudioFrames()) { - for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { - samples[idx] = frame.getMeanSampleAsDouble(sample); - } - } - return samples; + /** + * Returns the mean samples across all channels as short array. + * + * @return short array containing the mean sample values. + */ + default short[] getMeanSamplesAsShort() { + short[] samples = new short[this.getNumberOfSamples()]; + int idx = 0; + for (AudioFrame frame : this.getAudioFrames()) { + for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { + samples[idx] = frame.getMeanSampleAsShort(sample); + } } + return samples; + } - /** - * Returns the total number of samples in the frames segment (i.e. across all frames). - * - * @return Total number of samples in the segments - */ - default int getNumberOfSamples() { - return AudioFrame.EMPTY_FRAME.numberOfSamples(); + /** + * Returns the mean samples across all channels as double array. + * + * @return double array containing the mean sample values. + */ + default double[] getMeanSamplesAsDouble() { + double[] samples = new double[this.getNumberOfSamples()]; + int idx = 0; + for (AudioFrame frame : this.getAudioFrames()) { + for (int sample = 0; sample < frame.numberOfSamples(); sample++, idx++) { + samples[idx] = frame.getMeanSampleAsDouble(sample); + } } + return samples; + } - /** - * Returns the total duration in seconds of all samples in the frames segment - * (i.e. across all frames). - * - * @return Total duration in seconds. - */ - default float getAudioDuration() { - return AudioFrame.EMPTY_FRAME.getDuration(); - } + /** + * Returns the total number of samples in the frames segment (i.e. across all frames). + * + * @return Total number of samples in the segments + */ + default int getNumberOfSamples() { + return AudioFrame.EMPTY_FRAME.numberOfSamples(); + } - /** - * Returns the samplingrate of the segment. That rate usually determined by the first AudioFrame - * added to the segment and must be the same for all frames. - * - * @return Sampling rate of the frames segment. - */ - default float getSamplingrate() { - return AudioFrame.EMPTY_FRAME.getSamplingrate(); - } + /** + * Returns the total duration in seconds of all samples in the frames segment (i.e. across all frames). + * + * @return Total duration in seconds. + */ + default float getAudioDuration() { + return AudioFrame.EMPTY_FRAME.getDuration(); + } - /** - * Returns the number of channels for the frames segment. Is usually determined by - * the first AudioFrame added to the segment and must be the same for all frames. - * - * @return Number of channels of the frames segment. - */ - default int getChannels() { - return AudioFrame.EMPTY_FRAME.getChannels(); - } + /** + * Returns the samplingrate of the segment. That rate usually determined by the first AudioFrame added to the segment and must be the same for all frames. + * + * @return Sampling rate of the frames segment. + */ + default float getSamplingrate() { + return AudioFrame.EMPTY_FRAME.getSamplingrate(); + } + + /** + * Returns the number of channels for the frames segment. Is usually determined by the first AudioFrame added to the segment and must be the same for all frames. + * + * @return Number of channels of the frames segment. + */ + default int getChannels() { + return AudioFrame.EMPTY_FRAME.getChannels(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioSTFTProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioSTFTProvider.java index 88db4b47c..cea3ee9a1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioSTFTProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AudioSTFTProvider.java @@ -1,41 +1,37 @@ package org.vitrivr.cineast.core.data.providers; +import java.util.Arrays; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import java.util.Arrays; - public interface AudioSTFTProvider { - /** - * Calculates and returns the Short-term Fourier Transform of the - * current AudioSegment. - * - * @param windowsize Size of the window used during STFT. Must be a power of two. - * @param overlap Overlap in samples between two subsequent windows. - * @param function WindowFunction to apply before calculating the STFT. - * - * @return STFT of the current AudioSegment. - */ - default STFT getSTFT(int windowsize, int overlap, WindowFunction function) { - return this.getSTFT(windowsize, overlap, 0, function); - } - /** - * Calculates and returns the Short-term Fourier Transform of the - * current AudioSegment. - * - * @param windowsize Size of the window used during STFT. Must be a power of two. - * @param overlap Overlap in samples between two subsequent windows. - * @param function WindowFunction to apply before calculating the STFT. - * - * @return STFT of the current AudioSegment. - */ - default STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { - double[] data = new double[windowsize]; - Arrays.fill(data, 0.0); - STFT stft = new STFT(windowsize,overlap,padding,function, 22050); - stft.forward(data); - return stft; - } + /** + * Calculates and returns the Short-term Fourier Transform of the current AudioSegment. + * + * @param windowsize Size of the window used during STFT. Must be a power of two. + * @param overlap Overlap in samples between two subsequent windows. + * @param function WindowFunction to apply before calculating the STFT. + * @return STFT of the current AudioSegment. + */ + default STFT getSTFT(int windowsize, int overlap, WindowFunction function) { + return this.getSTFT(windowsize, overlap, 0, function); + } + + /** + * Calculates and returns the Short-term Fourier Transform of the current AudioSegment. + * + * @param windowsize Size of the window used during STFT. Must be a power of two. + * @param overlap Overlap in samples between two subsequent windows. + * @param function WindowFunction to apply before calculating the STFT. + * @return STFT of the current AudioSegment. + */ + default STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { + double[] data = new double[windowsize]; + Arrays.fill(data, 0.0); + STFT stft = new STFT(windowsize, overlap, padding, function, 22050); + stft.forward(data); + return stft; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AvgImgProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AvgImgProvider.java index 9ba6d779b..27ff32eb1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AvgImgProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/AvgImgProvider.java @@ -5,11 +5,10 @@ public interface AvgImgProvider { /** - * * @return the aggregated pixel-wise average of multiple images. By default, the {@link MultiImage}.EMPTY_MULTIIMAGE is returned. */ - public default MultiImage getAvgImg(){ - return MultiImage.EMPTY_MULTIIMAGE; - } - + public default MultiImage getAvgImg() { + return MultiImage.EMPTY_MULTIIMAGE; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/BooleanExpressionProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/BooleanExpressionProvider.java index 1123f5bec..b85f087f0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/BooleanExpressionProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/BooleanExpressionProvider.java @@ -1,13 +1,12 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.db.BooleanExpression; - import java.util.Collections; import java.util.List; +import org.vitrivr.cineast.core.db.BooleanExpression; public interface BooleanExpressionProvider { - default List getBooleanExpressions(){ + default List getBooleanExpressions() { return Collections.emptyList(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java index b066cf5c9..21c90870f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/DurationProvider.java @@ -1,47 +1,46 @@ package org.vitrivr.cineast.core.data.providers; public interface DurationProvider { - /** - * Returns the start in some arbitrary unit, e.g. in samples or frames depending - * on the implementation. - */ - default int getStart(){ - return 0; - } - /** - * Returns the end in some arbitrary unit, e.g. in samples or frames depending - * on the implementation. - */ - default int getEnd(){ - return 0; - } + /** + * Returns the start in some arbitrary unit, e.g. in samples or frames depending on the implementation. + */ + default int getStart() { + return 0; + } - /** - * Returns the relative start in percent (value between 0.0 and 1.0) - */ - default float getRelativeStart(){ - return 0f; - } + /** + * Returns the end in some arbitrary unit, e.g. in samples or frames depending on the implementation. + */ + default int getEnd() { + return 0; + } - /** - * Returns the relative end in percent (value between 0.0 and 1.0) - */ - default float getRelativeEnd(){ - return 0f; - } + /** + * Returns the relative start in percent (value between 0.0 and 1.0) + */ + default float getRelativeStart() { + return 0f; + } - /** - * Returns the absolute start in seconds. - */ - default float getAbsoluteStart(){ - return 0f; - } + /** + * Returns the relative end in percent (value between 0.0 and 1.0) + */ + default float getRelativeEnd() { + return 0f; + } - /** - * Returns the absolute end in seconds. - */ - default float getAbsoluteEnd(){ - return 0f; - } + /** + * Returns the absolute start in seconds. + */ + default float getAbsoluteStart() { + return 0f; + } + + /** + * Returns the absolute end in seconds. + */ + default float getAbsoluteEnd() { + return 0f; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/FrameListProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/FrameListProvider.java index 53de9670f..ebbc974af 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/FrameListProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/FrameListProvider.java @@ -1,15 +1,15 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.frames.VideoFrame; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.data.frames.VideoFrame; public interface FrameListProvider { - default List getVideoFrames() { - ArrayList list = new ArrayList<>(1); - list.add(VideoFrame.EMPTY_VIDEO_FRAME); - return list; - } + + default List getVideoFrames() { + ArrayList list = new ArrayList<>(1); + list.add(VideoFrame.EMPTY_VIDEO_FRAME); + return list; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/IdProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/IdProvider.java index 4b3798e74..b69810138 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/IdProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/IdProvider.java @@ -2,21 +2,22 @@ /** * An {@link IdProvider} does two things: It can give its own ID, and the ID of the larger thing it belongs to. - * + *

    * Therefore, implementing the {@link IdProvider} is a poor choice if a class can be part of multiple things. */ public interface IdProvider { - /** - * @return the identifier of this object - */ - String getId(); - /** - * @return the identifier of the larger group this object belongs to - */ - String getSuperId(); + /** + * @return the identifier of this object + */ + String getId(); - void setId(String id); + /** + * @return the identifier of the larger group this object belongs to + */ + String getSuperId(); - void setSuperId(String id); + void setId(String id); + + void setSuperId(String id); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/InstantProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/InstantProvider.java index 8f9306767..add000270 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/InstantProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/InstantProvider.java @@ -4,6 +4,7 @@ import java.util.Optional; public interface InstantProvider { + default Optional getInstant() { return Optional.empty(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/LocationProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/LocationProvider.java index 30d386607..e82ca0bee 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/LocationProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/LocationProvider.java @@ -1,10 +1,10 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.Location; - import java.util.Optional; +import org.vitrivr.cineast.core.data.Location; public interface LocationProvider { + default Optional getLocation() { return Optional.empty(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MedianImgProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MedianImgProvider.java index 63c45f001..f57f981d0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MedianImgProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MedianImgProvider.java @@ -5,11 +5,10 @@ public interface MedianImgProvider { /** - * * @return the aggregated pixel-wise median of multiple images. By default, the {@link MultiImage}.EMPTY_MULTIIMAGE is returned. */ - public default MultiImage getMedianImg(){ - return MultiImage.EMPTY_MULTIIMAGE; - } - + public default MultiImage getMedianImg() { + return MultiImage.EMPTY_MULTIIMAGE; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MeshProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MeshProvider.java index f397d4a8c..6bbc3a24c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MeshProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MeshProvider.java @@ -6,49 +6,46 @@ public interface MeshProvider { - /** - * Returns the original 3D Mesh. Defaults to the empty mesh, if the method - * hasn't been implemented. - * - * Important: The instance of the Mesh returned by this method is - * potentially shared by different feature modules and Threads. Keep that in mind when - * applying in-place transformations on the Mesh! - * - * @return Original mesh. - */ - default ReadableMesh getMesh() { - return Mesh.EMPTY; - } - /** - * Returns a copy of the original 3D Mesh. - * - * @return Copy of the original 3D mesh. - */ - default WritableMesh copyMesh() { - return new Mesh(this.getMesh()); - } + /** + * Returns the original 3D Mesh. Defaults to the empty mesh, if the method hasn't been implemented. + * + * Important: The instance of the Mesh returned by this method is + * potentially shared by different feature modules and Threads. Keep that in mind when applying in-place transformations on the Mesh! + * + * @return Original mesh. + */ + default ReadableMesh getMesh() { + return Mesh.EMPTY; + } - /** - * Returns a version of the 3D Mesh that has been KHL-transformed. Defaults to the empty mesh, - * if the method hasn't been implemented. - * - * Important: The instance of the Mesh returned by this method is potentially - * shared by different feature modules and Threads. Keep that in mind when applying in-place - * transformations on the Mesh! - * - * @return KHL transformed mesh. - */ - default ReadableMesh getNormalizedMesh() { - return Mesh.EMPTY; - } + /** + * Returns a copy of the original 3D Mesh. + * + * @return Copy of the original 3D mesh. + */ + default WritableMesh copyMesh() { + return new Mesh(this.getMesh()); + } - /** - * Returns a copy of the KHL transformed 3D Mesh. - * - * @return Copy of the KHL transformed 3D mesh. - */ - default WritableMesh copyNormalizedMesh() { - return new Mesh(this.getNormalizedMesh()); - } + /** + * Returns a version of the 3D Mesh that has been KHL-transformed. Defaults to the empty mesh, if the method hasn't been implemented. + * + * Important: The instance of the Mesh returned by this method is potentially + * shared by different feature modules and Threads. Keep that in mind when applying in-place transformations on the Mesh! + * + * @return KHL transformed mesh. + */ + default ReadableMesh getNormalizedMesh() { + return Mesh.EMPTY; + } + + /** + * Returns a copy of the KHL transformed 3D Mesh. + * + * @return Copy of the KHL transformed 3D mesh. + */ + default WritableMesh copyNormalizedMesh() { + return new Mesh(this.getNormalizedMesh()); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MostRepresentativeFrameProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MostRepresentativeFrameProvider.java index 461813682..07fc83656 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MostRepresentativeFrameProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/MostRepresentativeFrameProvider.java @@ -3,13 +3,12 @@ import org.vitrivr.cineast.core.data.frames.VideoFrame; public interface MostRepresentativeFrameProvider { - + /** - * * @return the frame which best represents a sequence of frames. By default, the {@link VideoFrame}.EMPTY_FRAME is returned */ - public default VideoFrame getMostRepresentativeFrame(){ - return VideoFrame.EMPTY_VIDEO_FRAME; - } - + public default VideoFrame getMostRepresentativeFrame() { + return VideoFrame.EMPTY_VIDEO_FRAME; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/PathProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/PathProvider.java index d84fe8024..75454a3e4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/PathProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/PathProvider.java @@ -1,21 +1,20 @@ package org.vitrivr.cineast.core.data.providers; import georegression.struct.point.Point2D_F32; -import org.vitrivr.cineast.core.data.Pair; - import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import org.vitrivr.cineast.core.data.Pair; public interface PathProvider { - public default List>> getPaths(){ - return new ArrayList<>(0); - } - - public default List>> getBgPaths(){ - return new ArrayList<>(0); - } - + public default List>> getPaths() { + return new ArrayList<>(0); + } + + public default List>> getBgPaths() { + return new ArrayList<>(0); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SegmentProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SegmentProvider.java index 706bcc109..82766c758 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SegmentProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SegmentProvider.java @@ -5,12 +5,11 @@ public interface SegmentProvider extends AutoCloseable { /** - * * @return the next {@link SegmentContainer} from the source or null if there are no more segments. */ - public SegmentContainer getNextSegment(); + public SegmentContainer getNextSegment(); - @Override + @Override public void close(); - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SemanticMapProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SemanticMapProvider.java index 453e10de8..85bcb41bb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SemanticMapProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SemanticMapProvider.java @@ -1,13 +1,12 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.SemanticMap; - import java.util.Optional; +import org.vitrivr.cineast.core.data.SemanticMap; public interface SemanticMapProvider { - default Optional getSemanticMap(){ - return Optional.empty(); - } + default Optional getSemanticMap() { + return Optional.empty(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SubtitleItemProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SubtitleItemProvider.java index 610821367..25d74c1fc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SubtitleItemProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SubtitleItemProvider.java @@ -1,14 +1,13 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; public interface SubtitleItemProvider { - public default List getSubtitleItems(){ - return new ArrayList<>(0); - } - + public default List getSubtitleItems() { + return new ArrayList<>(0); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TagProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TagProvider.java index ec188890e..d1cfb0e62 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TagProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TagProvider.java @@ -1,14 +1,13 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.tag.Tag; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.data.tag.Tag; public interface TagProvider { - public default List getTags(){ - return new ArrayList<>(0); - } - + public default List getTags() { + return new ArrayList<>(0); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TextProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TextProvider.java index 81039fcc0..b2b5d08b9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TextProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/TextProvider.java @@ -2,8 +2,8 @@ public interface TextProvider { - public default String getText(){ + public default String getText() { return ""; } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/VoxelGridProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/VoxelGridProvider.java index 9cf5678ff..2764bf0cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/VoxelGridProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/VoxelGridProvider.java @@ -4,13 +4,13 @@ public interface VoxelGridProvider { - /** - * Returns a 3D VoxelGrid. Defaults to the empty VoxelGrid, - * if not implemented. - * - * @return VoxelGrid - */ - default VoxelGrid getVoxelgrid() { - return VoxelGrid.EMPTY; - } + + /** + * Returns a 3D VoxelGrid. Defaults to the empty VoxelGrid, if not implemented. + * + * @return VoxelGrid + */ + default VoxelGrid getVoxelgrid() { + return VoxelGrid.EMPTY; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetProvider.java index 229b25381..1c9fd4766 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetProvider.java @@ -4,6 +4,7 @@ public interface BitSetProvider { + default BitSet getBitSet() { throw new UnsupportedOperationException("No BitSet specified"); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetTypeProvider.java index 376e170df..06600ffef 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BitSetTypeProvider.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.core.data.providers.primitive; import com.googlecode.javaewah.datastructure.BitSet; - import java.util.Arrays; import java.util.List; @@ -18,23 +17,23 @@ public BitSet getValue() { } - public static BitSetTypeProvider fromBooleanList(List list){ + public static BitSetTypeProvider fromBooleanList(List list) { final BitSet bitSet = new BitSet(list.size()); - for (int i = 0; i< list.size(); i++) { + for (int i = 0; i < list.size(); i++) { bitSet.set(i, list.get(i)); } return new BitSetTypeProvider(bitSet); } - public static BitSetTypeProvider fromBooleanArray(boolean[] array){ + public static BitSetTypeProvider fromBooleanArray(boolean[] array) { final BitSet bitSet = new BitSet(array.length); - for (int i = 0; i< array.length; i++) { + for (int i = 0; i < array.length; i++) { bitSet.set(i, array[i]); } return new BitSetTypeProvider(bitSet); } - public static BitSetTypeProvider fromString(String string){ + public static BitSetTypeProvider fromString(String string) { final String raw = string.substring(1, string.length() - 1); final BitSet bitSet = new BitSet(64); //TODO We assume fixed size here Arrays.stream(raw.split(",")).forEach(el -> bitSet.set(Integer.parseInt(el))); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProvider.java index 9766bca4d..0f57a0f7e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface BooleanProvider { - - default boolean getBoolean(){ - throw new UnsupportedOperationException("No boolean value specified"); - } - + + default boolean getBoolean() { + throw new UnsupportedOperationException("No boolean value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProviderImpl.java index b2dd07030..9a26a0752 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanProviderImpl.java @@ -2,45 +2,45 @@ public class BooleanProviderImpl implements BooleanProvider { - private final boolean value; - - public BooleanProviderImpl(boolean value){ - this.value = value; - } - - @Override - public boolean getBoolean() { - return this.value; - } + private final boolean value; - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (value ? 1231 : 1237); - return result; - } + public BooleanProviderImpl(boolean value) { + this.value = value; + } - @Override - public boolean equals(Object obj) { - if (this == obj) { + @Override + public boolean getBoolean() { + return this.value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (value ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - BooleanProviderImpl other = (BooleanProviderImpl) obj; - if (value != other.value) { + BooleanProviderImpl other = (BooleanProviderImpl) obj; + if (value != other.value) { return false; } - return true; - } + return true; + } - @Override - public String toString() { - return "BooleanProviderImpl [value=" + value + "]"; - } + @Override + public String toString() { + return "BooleanProviderImpl [value=" + value + "]"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanTypeProvider.java index 10ec92c8a..dbc5769ab 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/BooleanTypeProvider.java @@ -1,19 +1,19 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class BooleanTypeProvider extends BooleanProviderImpl implements PrimitiveTypeProvider{ +public class BooleanTypeProvider extends BooleanProviderImpl implements PrimitiveTypeProvider { - public BooleanTypeProvider(boolean value) { - super(value); - } + public BooleanTypeProvider(boolean value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.BOOLEAN; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.BOOLEAN; - } - - @Override + @Override public String getString() { return getBoolean() ? "true" : "false"; } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProvider.java index 81ac867ee..5d0b39f6b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProvider.java @@ -2,8 +2,8 @@ public interface ByteProvider { - default byte getByte(){ - throw new UnsupportedOperationException("No byte value specified"); - } - + default byte getByte() { + throw new UnsupportedOperationException("No byte value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProviderImpl.java index 407e18c88..a1db3e414 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteProviderImpl.java @@ -2,18 +2,18 @@ public class ByteProviderImpl implements ByteProvider, ShortProvider, IntProvider, LongProvider { - private final byte value; - - public ByteProviderImpl(byte value){ - this.value = value; - } - - @Override - public byte getByte() { - return this.value; - } - - @Override + private final byte value; + + public ByteProviderImpl(byte value) { + this.value = value; + } + + @Override + public byte getByte() { + return this.value; + } + + @Override public short getShort() { return this.value; } @@ -28,34 +28,34 @@ public long getLong() { return this.value; } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + value; - return result; - } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + value; + return result; + } - @Override - public boolean equals(Object obj) { - if (this == obj) { + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - ByteProviderImpl other = (ByteProviderImpl) obj; - if (value != other.value) { + ByteProviderImpl other = (ByteProviderImpl) obj; + if (value != other.value) { return false; } - return true; - } + return true; + } - @Override - public String toString() { - return "ByteProviderImpl [value=" + value + "]"; - } + @Override + public String toString() { + return "ByteProviderImpl [value=" + value + "]"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteTypeProvider.java index 1dfa1f1fa..7c918a016 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ByteTypeProvider.java @@ -10,7 +10,7 @@ public ByteTypeProvider(byte value) { public ProviderDataType getType() { return ProviderDataType.BYTE; } - + @Override public String getString() { return Byte.toString(getByte()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProvider.java index 5acf4d820..20811742c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface DoubleProvider { - - default double getDouble(){ - throw new UnsupportedOperationException("No double value specified"); - } - + + default double getDouble() { + throw new UnsupportedOperationException("No double value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProviderImpl.java index 28eda622f..0e1042fc4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleProviderImpl.java @@ -2,53 +2,53 @@ public class DoubleProviderImpl implements DoubleProvider, FloatProvider { - private final double value; - - public DoubleProviderImpl(double value){ - this.value = value; - } - - @Override - public double getDouble() { - return this.value; - } - - @Override - public float getFloat() { - return (float)this.value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - long temp; - temp = Double.doubleToLongBits(value); - result = prime * result + (int) (temp ^ (temp >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + private final double value; + + public DoubleProviderImpl(double value) { + this.value = value; + } + + @Override + public double getDouble() { + return this.value; + } + + @Override + public float getFloat() { + return (float) this.value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(value); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - DoubleProviderImpl other = (DoubleProviderImpl) obj; - if (Double.doubleToLongBits(value) != Double.doubleToLongBits(other.value)) { + DoubleProviderImpl other = (DoubleProviderImpl) obj; + if (Double.doubleToLongBits(value) != Double.doubleToLongBits(other.value)) { return false; } - return true; - } - - @Override - public String toString() { - return "DoubleProviderImpl [value=" + value + "]"; - } - + return true; + } + + @Override + public String toString() { + return "DoubleProviderImpl [value=" + value + "]"; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleTypeProvider.java index ce0e78042..518adb162 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/DoubleTypeProvider.java @@ -1,19 +1,19 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class DoubleTypeProvider extends DoubleProviderImpl implements PrimitiveTypeProvider{ +public class DoubleTypeProvider extends DoubleProviderImpl implements PrimitiveTypeProvider { - public DoubleTypeProvider(double value) { - super(value); - } + public DoubleTypeProvider(double value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.DOUBLE; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.DOUBLE; - } - - @Override + @Override public String getString() { return Double.toString(getDouble()); } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProvider.java index 0cb6a8e42..e6180a4d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProvider.java @@ -3,9 +3,9 @@ public interface FloatArrayProvider { public static final float[] DEFAULT_FLOAT_ARRAY = new float[]{}; - - default float[] getFloatArray(){ - throw new UnsupportedOperationException("No float array specified"); - } - + + default float[] getFloatArray() { + throw new UnsupportedOperationException("No float array specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProviderImpl.java index bfbff17b5..2d18d6dd7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayProviderImpl.java @@ -4,49 +4,49 @@ public class FloatArrayProviderImpl implements FloatArrayProvider { - private final float[] value; - - public FloatArrayProviderImpl(float[] value){ - if(value == null){ - throw new NullPointerException("float[] cannot be null"); - } - this.value = value; - } - - @Override - public float[] getFloatArray() { - return this.value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(value); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + private final float[] value; + + public FloatArrayProviderImpl(float[] value) { + if (value == null) { + throw new NullPointerException("float[] cannot be null"); + } + this.value = value; + } + + @Override + public float[] getFloatArray() { + return this.value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(value); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - FloatArrayProviderImpl other = (FloatArrayProviderImpl) obj; - if (!Arrays.equals(value, other.value)) { + FloatArrayProviderImpl other = (FloatArrayProviderImpl) obj; + if (!Arrays.equals(value, other.value)) { return false; } - return true; - } - - @Override - public String toString() { - return "FloatArrayProviderImpl [value=" + Arrays.toString(value) + "]"; - } - + return true; + } + + @Override + public String toString() { + return "FloatArrayProviderImpl [value=" + Arrays.toString(value) + "]"; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayTypeProvider.java index c6da8f4f6..909facd45 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatArrayTypeProvider.java @@ -14,18 +14,18 @@ public FloatArrayTypeProvider(float[] value) { public ProviderDataType getType() { return ProviderDataType.FLOAT_ARRAY; } - + @Override public String getString() { return Arrays.toString(getFloatArray()); } - public static FloatArrayTypeProvider fromList(List list){ + public static FloatArrayTypeProvider fromList(List list) { float[] array = new float[list.size()]; int i = 0; - for(Float f : list){ + for (Float f : list) { array[i++] = f; } @@ -40,7 +40,7 @@ public static PrimitiveTypeProvider fromDoubleList(List list) { int i = 0; - for(Double d : list){ + for (Double d : list) { array[i++] = d.floatValue(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProvider.java index f39d816f3..1aacd772d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface FloatProvider { - - default float getFloat(){ - throw new UnsupportedOperationException("No float value specified"); - } - + + default float getFloat() { + throw new UnsupportedOperationException("No float value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProviderImpl.java index 7414b040b..2140b08a0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatProviderImpl.java @@ -2,51 +2,51 @@ public class FloatProviderImpl implements FloatProvider, DoubleProvider { - private final float value; - - public FloatProviderImpl(float value){ - this.value = value; - } - - @Override - public float getFloat() { - return this.value; - } - - @Override + private final float value; + + public FloatProviderImpl(float value) { + this.value = value; + } + + @Override + public float getFloat() { + return this.value; + } + + @Override public double getDouble() { return this.value; } @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Float.floatToIntBits(value); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Float.floatToIntBits(value); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - FloatProviderImpl other = (FloatProviderImpl) obj; - if (Float.floatToIntBits(value) != Float.floatToIntBits(other.value)) { + FloatProviderImpl other = (FloatProviderImpl) obj; + if (Float.floatToIntBits(value) != Float.floatToIntBits(other.value)) { return false; } - return true; - } - - @Override - public String toString() { - return "FloatProviderImpl [value=" + value + "]"; - } - + return true; + } + + @Override + public String toString() { + return "FloatProviderImpl [value=" + value + "]"; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatTypeProvider.java index 8e9d1431e..c8675a8da 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatTypeProvider.java @@ -1,19 +1,19 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class FloatTypeProvider extends FloatProviderImpl implements PrimitiveTypeProvider{ +public class FloatTypeProvider extends FloatProviderImpl implements PrimitiveTypeProvider { - public FloatTypeProvider(float value) { - super(value); - } + public FloatTypeProvider(float value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.FLOAT; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.FLOAT; - } - - @Override + @Override public String getString() { return Float.toString(getFloat()); } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatVectorProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatVectorProvider.java index c004fdd02..2a5477813 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatVectorProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/FloatVectorProvider.java @@ -4,16 +4,16 @@ public class FloatVectorProvider extends FloatArrayProviderImpl implements PrimitiveTypeProvider { - public FloatVectorProvider(float[] array) { - super(array); - } + public FloatVectorProvider(float[] array) { + super(array); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.FLOAT_ARRAY; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.FLOAT_ARRAY; - } - - @Override + @Override public String getString() { return Arrays.toString(getFloatArray()); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProvider.java index 616f0ea50..b85761518 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProvider.java @@ -2,10 +2,10 @@ public interface IntArrayProvider { - public static final int[] DEFAULT_INT_ARRAY = new int[]{}; - - default int[] getIntArray(){ - throw new UnsupportedOperationException("No int array specified"); - } - + public static final int[] DEFAULT_INT_ARRAY = new int[]{}; + + default int[] getIntArray() { + throw new UnsupportedOperationException("No int array specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProviderImpl.java index c93a150a2..d20466e75 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayProviderImpl.java @@ -4,20 +4,20 @@ public class IntArrayProviderImpl implements IntArrayProvider, FloatArrayProvider { - private final int[] value; + private final int[] value; private float[] fvalue = null; - - public IntArrayProviderImpl(int[] value){ - if(value == null){ - throw new NullPointerException("int[] cannot be null"); - } - this.value = value; - } - @Override - public int[] getIntArray() { - return this.value; - } + public IntArrayProviderImpl(int[] value) { + if (value == null) { + throw new NullPointerException("int[] cannot be null"); + } + this.value = value; + } + + @Override + public int[] getIntArray() { + return this.value; + } @Override public float[] getFloatArray() { @@ -32,35 +32,35 @@ public float[] getFloatArray() { return this.fvalue; } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(value); - return result; - } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(value); + return result; + } - @Override - public boolean equals(Object obj) { - if (this == obj) { + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - IntArrayProviderImpl other = (IntArrayProviderImpl) obj; - if (!Arrays.equals(value, other.value)) { + IntArrayProviderImpl other = (IntArrayProviderImpl) obj; + if (!Arrays.equals(value, other.value)) { return false; } - return true; - } + return true; + } + + @Override + public String toString() { + return String.format("IntArrayProviderImpl [value=%s]", Arrays.toString(value)); + } - @Override - public String toString() { - return String.format("IntArrayProviderImpl [value=%s]", Arrays.toString(value)); - } - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayTypeProvider.java index a61c09900..21a3bdb36 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntArrayTypeProvider.java @@ -10,7 +10,6 @@ public IntArrayTypeProvider(int[] value) { } - @Override public ProviderDataType getType() { return ProviderDataType.INT_ARRAY; @@ -28,7 +27,7 @@ public static PrimitiveTypeProvider fromList(List list) { int i = 0; - for(Integer in : list){ + for (Integer in : list) { array[i++] = in; } @@ -41,7 +40,7 @@ public static PrimitiveTypeProvider fromLongList(List list) { int i = 0; - for(Long l : list){ + for (Long l : list) { array[i++] = l.intValue(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProvider.java index 81a657799..c9c0c6043 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface IntProvider { - - default int getInt(){ - throw new UnsupportedOperationException("No int value specified"); - } - + + default int getInt() { + throw new UnsupportedOperationException("No int value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProviderImpl.java index 9141c6b97..56b2ab252 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntProviderImpl.java @@ -2,61 +2,61 @@ public class IntProviderImpl implements IntProvider, LongProvider, FloatProvider, DoubleProvider { - private final int value; - - public IntProviderImpl(int value){ - this.value = value; - } - - @Override - public int getInt() { - return this.value; - } - - @Override + private final int value; + + public IntProviderImpl(int value) { + this.value = value; + } + + @Override + public int getInt() { + return this.value; + } + + @Override public long getLong() { return this.value; } @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + value; - return result; - } - - @Override - public double getDouble() { - return value; - } - - @Override - public float getFloat() { - return value; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + value; + return result; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public float getFloat() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - IntProviderImpl other = (IntProviderImpl) obj; - if (value != other.value) { + IntProviderImpl other = (IntProviderImpl) obj; + if (value != other.value) { return false; } - return true; - } - - @Override - public String toString() { - return String.format("IntProviderImpl [value=%s]", value); - } - + return true; + } + + @Override + public String toString() { + return String.format("IntProviderImpl [value=%s]", value); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntTypeProvider.java index cedc36246..6bbd3f3a1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntTypeProvider.java @@ -1,19 +1,19 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class IntTypeProvider extends IntProviderImpl implements PrimitiveTypeProvider{ +public class IntTypeProvider extends IntProviderImpl implements PrimitiveTypeProvider { - public IntTypeProvider(int value) { - super(value); - } + public IntTypeProvider(int value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.INT; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.INT; - } - - @Override + @Override public String getString() { return Integer.toString(getInt()); } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntVectorProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntVectorProvider.java index c86342d50..23e473d74 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntVectorProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/IntVectorProvider.java @@ -2,20 +2,20 @@ import java.util.Arrays; -public class IntVectorProvider extends IntArrayProviderImpl implements PrimitiveTypeProvider{ +public class IntVectorProvider extends IntArrayProviderImpl implements PrimitiveTypeProvider { - public IntVectorProvider(int[] array) { - super(array); - } + public IntVectorProvider(int[] array) { + super(array); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.INT_ARRAY; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.INT_ARRAY; - } - - @Override + @Override public String getString() { return Arrays.toString(getIntArray()); } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProvider.java index 5056929db..d28e7cd7b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface LongProvider { - - default long getLong(){ - throw new UnsupportedOperationException("No long value specified"); - } - + + default long getLong() { + throw new UnsupportedOperationException("No long value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProviderImpl.java index d66918688..196097283 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongProviderImpl.java @@ -2,50 +2,50 @@ public class LongProviderImpl implements LongProvider, DoubleProvider { - private final long value; - - public LongProviderImpl(long value){ - this.value = value; - } - - @Override - public long getLong() { - return this.value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (int) (value ^ (value >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + private final long value; + + public LongProviderImpl(long value) { + this.value = value; + } + + @Override + public long getLong() { + return this.value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (value ^ (value >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - LongProviderImpl other = (LongProviderImpl) obj; - if (value != other.value) { + LongProviderImpl other = (LongProviderImpl) obj; + if (value != other.value) { return false; } - return true; - } - - @Override - public String toString() { - return String.format("LongProviderImpl [value=%s]", value); - } - - @Override - public double getDouble() { - return value; - } + return true; + } + + @Override + public String toString() { + return String.format("LongProviderImpl [value=%s]", value); + } + + @Override + public double getDouble() { + return value; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongTypeProvider.java index 15a47f1ce..e031be3a9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/LongTypeProvider.java @@ -1,19 +1,19 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class LongTypeProvider extends LongProviderImpl implements PrimitiveTypeProvider{ +public class LongTypeProvider extends LongProviderImpl implements PrimitiveTypeProvider { - public LongTypeProvider(long value) { - super(value); - } + public LongTypeProvider(long value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.LONG; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.LONG; - } - - @Override + @Override public String getString() { return Long.toString(getLong()); } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/NothingProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/NothingProvider.java index 041f0a43a..2cd208b9f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/NothingProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/NothingProvider.java @@ -1,17 +1,17 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class NothingProvider implements PrimitiveTypeProvider{ +public class NothingProvider implements PrimitiveTypeProvider { public static final NothingProvider INSTANCE = new NothingProvider(); - - @Override - public ProviderDataType getType() { - return ProviderDataType.UNKNOWN; - } + + @Override + public ProviderDataType getType() { + return ProviderDataType.UNKNOWN; + } @Override public String toString() { return "NothingProvider"; } - + } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveProviderComparator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveProviderComparator.java index ca1d73860..6a1894a3e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveProviderComparator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveProviderComparator.java @@ -7,26 +7,26 @@ public class PrimitiveProviderComparator implements ComparatornaturalOrder().compare(o1.getString(), o2.getString()); } return -1; - }else if(o2.getType() == ProviderDataType.STRING){ + } else if (o2.getType() == ProviderDataType.STRING) { return 1; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveTypeProvider.java index efdb095b4..1eb46b97c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/PrimitiveTypeProvider.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.googlecode.javaewah.datastructure.BitSet; - import java.util.List; public interface PrimitiveTypeProvider diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ProviderDataType.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ProviderDataType.java index 88ecdd8bd..a2e0cb12f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ProviderDataType.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ProviderDataType.java @@ -1,7 +1,7 @@ package org.vitrivr.cineast.core.data.providers.primitive; public enum ProviderDataType { - + BOOLEAN, BYTE, SHORT, @@ -13,5 +13,5 @@ public enum ProviderDataType { DOUBLE, STRING, UNKNOWN, BITSET; - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProvider.java index ca5cf76e4..f6e59fa08 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface ShortProvider { - - default short getShort(){ - throw new UnsupportedOperationException("No short value specified"); - } - + + default short getShort() { + throw new UnsupportedOperationException("No short value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProviderImpl.java index 830bfa743..7727d19cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortProviderImpl.java @@ -2,18 +2,18 @@ public class ShortProviderImpl implements ShortProvider, IntProvider, LongProvider { - private final short value; - - public ShortProviderImpl(short value){ - this.value = value; - } - - @Override - public short getShort() { - return this.value; - } - - @Override + private final short value; + + public ShortProviderImpl(short value) { + this.value = value; + } + + @Override + public short getShort() { + return this.value; + } + + @Override public int getInt() { return this.value; } @@ -24,33 +24,33 @@ public long getLong() { } @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + value; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + value; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - ShortProviderImpl other = (ShortProviderImpl) obj; - if (value != other.value) { + ShortProviderImpl other = (ShortProviderImpl) obj; + if (value != other.value) { return false; } - return true; - } + return true; + } - @Override - public String toString() { - return String.format("ShortProviderImpl [value=%s]", value); - } + @Override + public String toString() { + return String.format("ShortProviderImpl [value=%s]", value); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortTypeProvider.java index f1e0b7992..9de88ca9e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/ShortTypeProvider.java @@ -8,9 +8,9 @@ public ShortTypeProvider(short value) { @Override public ProviderDataType getType() { - return ProviderDataType.SHORT; + return ProviderDataType.SHORT; } - + @Override public String getString() { return Short.toString(getShort()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProvider.java index 17066d54f..13c558f94 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProvider.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.data.providers.primitive; public interface StringProvider { - - default String getString(){ - throw new UnsupportedOperationException("No string value specified"); - } - + + default String getString() { + throw new UnsupportedOperationException("No string value specified"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProviderImpl.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProviderImpl.java index c0839bbe1..38ae47cc0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProviderImpl.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringProviderImpl.java @@ -2,52 +2,52 @@ public class StringProviderImpl implements StringProvider { - private final String value; - - public StringProviderImpl(String value){ - if(value == null){ - throw new NullPointerException("String cannot be null"); - } - this.value = value; - } + private final String value; - @Override - public String getString() { - return this.value; - } + public StringProviderImpl(String value) { + if (value == null) { + throw new NullPointerException("String cannot be null"); + } + this.value = value; + } + + @Override + public String getString() { + return this.value; + } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } - @Override - public boolean equals(Object obj) { - if (this == obj) { + @Override + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (getClass() != obj.getClass()) { return false; } - StringProviderImpl other = (StringProviderImpl) obj; - if (value == null) { - if (other.value != null) { + StringProviderImpl other = (StringProviderImpl) obj; + if (value == null) { + if (other.value != null) { return false; } - } else if (!value.equals(other.value)) { + } else if (!value.equals(other.value)) { return false; } - return true; - } + return true; + } - @Override - public String toString() { - return String.format("StringProviderImpl [value=%s]", value); - } + @Override + public String toString() { + return String.format("StringProviderImpl [value=%s]", value); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringTypeProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringTypeProvider.java index 1bc7934c5..bc54f54e1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringTypeProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/primitive/StringTypeProvider.java @@ -1,14 +1,14 @@ package org.vitrivr.cineast.core.data.providers.primitive; -public class StringTypeProvider extends StringProviderImpl implements PrimitiveTypeProvider{ +public class StringTypeProvider extends StringProviderImpl implements PrimitiveTypeProvider { - public StringTypeProvider(String value) { - super(value); - } + public StringTypeProvider(String value) { + super(value); + } + + @Override + public ProviderDataType getType() { + return ProviderDataType.STRING; + } - @Override - public ProviderDataType getType() { - return ProviderDataType.STRING; - } - } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AbstractQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AbstractQueryTermContainer.java index 5966b7dd5..a55fcff92 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AbstractQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AbstractQueryTermContainer.java @@ -9,7 +9,7 @@ /** * An {@link AbstractQueryTermContainer} is the implementation of a {@link SegmentContainer} which is used in the online-phase (during retrieval). - * + *

    * On a system perspective, it is generally created based on an API request from a Query Term (e.g. a color sketch, a text query). */ public abstract class AbstractQueryTermContainer implements SegmentContainer { @@ -39,7 +39,7 @@ public void setWeight(float weight) { /** * Online: If this is set, it is assumed to be the id of a {@link MediaSegmentDescriptor} - * + *

    * Offline: Id of the segment whose extraction is running */ @Override @@ -64,7 +64,7 @@ public void setSuperId(String id) { /** * Online: If this is set, it is assumed to be the id of a {@link MediaObjectDescriptor} - * + *

    * Offline: The {@link MediaObjectDescriptor} this segment belongs to. */ @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java index e4f58d854..031239c37 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/AudioQueryTermContainer.java @@ -1,97 +1,100 @@ package org.vitrivr.cineast.core.data.query.containers; +import java.util.List; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; import org.vitrivr.cineast.core.util.web.AudioParser; -import java.util.List; - public class AudioQueryTermContainer extends AbstractQueryTermContainer { - /** List of {@link AudioFrame}s. */ - private final List frames; + /** + * List of {@link AudioFrame}s. + */ + private final List frames; - /** Total number of samples in the AudioSegment. */ - private int totalSamples; + /** + * Total number of samples in the AudioSegment. + */ + private int totalSamples; - /** Total duration of the AudioSegment in seconds. */ - private float totalDuration; + /** + * Total duration of the AudioSegment in seconds. + */ + private float totalDuration; - /** {@link AudioDescriptor} describing the properties of the underlying audio stream. */ - private AudioDescriptor descriptor; + /** + * {@link AudioDescriptor} describing the properties of the underlying audio stream. + */ + private AudioDescriptor descriptor; - /** - * Constructs an {@link AudioQueryTermContainer} from base 64 encoded wave audio data. The constructor assumes - * the following audio settings: 22050Hz, 1 Channel, 16bit PCM - * - * @param data The audio data that should be converted. - */ - public AudioQueryTermContainer(String data) { - this(AudioParser.parseWaveAudio(data, 22050.0f, 1)); - } + /** + * Constructs an {@link AudioQueryTermContainer} from base 64 encoded wave audio data. The constructor assumes the following audio settings: 22050Hz, 1 Channel, 16bit PCM + * + * @param data The audio data that should be converted. + */ + public AudioQueryTermContainer(String data) { + this(AudioParser.parseWaveAudio(data, 22050.0f, 1)); + } - /** - * Returns a list of audio-frames contained in the AudioSegment. The default implementation returns - * a list containing one, empty frame. - * - * List auf audio-frames in the audio-segment. - */ - public AudioQueryTermContainer(List frames) { - this.frames = frames; - for (AudioFrame frame : this.frames) { - if (this.descriptor == null) { - this.descriptor = frame.getDescriptor(); - } - if (!this.descriptor.equals(frame.getDescriptor())) { - throw new IllegalArgumentException("All the provided AudioFrames must share the same AudioDescriptor!"); - } - this.totalSamples += frame.numberOfSamples(); - this.totalDuration += frame.getDuration(); - } + /** + * Returns a list of audio-frames contained in the AudioSegment. The default implementation returns a list containing one, empty frame. + *

    + * List auf audio-frames in the audio-segment. + */ + public AudioQueryTermContainer(List frames) { + this.frames = frames; + for (AudioFrame frame : this.frames) { + if (this.descriptor == null) { + this.descriptor = frame.getDescriptor(); + } + if (!this.descriptor.equals(frame.getDescriptor())) { + throw new IllegalArgumentException("All the provided AudioFrames must share the same AudioDescriptor!"); + } + this.totalSamples += frame.numberOfSamples(); + this.totalDuration += frame.getDuration(); } + } - @Override - public List getAudioFrames() { - return this.frames; - } + @Override + public List getAudioFrames() { + return this.frames; + } - /** - * Getter for the total number of samples in the AudioSegment. - */ - @Override - public int getNumberOfSamples() { - return this.totalSamples; - } + /** + * Getter for the total number of samples in the AudioSegment. + */ + @Override + public int getNumberOfSamples() { + return this.totalSamples; + } - /** - * Getter for the total duration of the AudioSegment. - */ - @Override - public float getAudioDuration() { - return totalDuration; - } + /** + * Getter for the total duration of the AudioSegment. + */ + @Override + public float getAudioDuration() { + return totalDuration; + } - /** - * Calculates and returns the Short-term Fourier Transform of the - * current AudioSegment. - * - * @param windowsize Size of the window used during STFT. Must be a power of two. - * @param overlap Overlap in samples between two subsequent windows. - * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. - * @param function WindowFunction to apply before calculating the STFT. - * - * @return STFT of the current AudioSegment. - */ - @Override - public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { - if (2*padding >= windowsize) { - throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); - } - STFT stft = new STFT(windowsize, overlap, padding, function, this.descriptor.getSamplingrate()); - stft.forward(this.getMeanSamplesAsDouble()); - return stft; + /** + * Calculates and returns the Short-term Fourier Transform of the current AudioSegment. + * + * @param windowsize Size of the window used during STFT. Must be a power of two. + * @param overlap Overlap in samples between two subsequent windows. + * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. + * @param function WindowFunction to apply before calculating the STFT. + * @return STFT of the current AudioSegment. + */ + @Override + public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { + if (2 * padding >= windowsize) { + throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); } + STFT stft = new STFT(windowsize, overlap, padding, function, this.descriptor.getSamplingrate()); + stft.forward(this.getMeanSamplesAsDouble()); + return stft; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/BooleanQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/BooleanQueryTermContainer.java index b3109388e..c0b51d160 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/BooleanQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/BooleanQueryTermContainer.java @@ -1,6 +1,10 @@ package org.vitrivr.cineast.core.data.query.containers; import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; @@ -8,11 +12,6 @@ import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.util.web.DataURLParser; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; - public class BooleanQueryTermContainer extends AbstractQueryTermContainer { private static final String ATTRIBUTE_FIELD_NAME = "attribute"; @@ -64,7 +63,7 @@ public BooleanQueryTermContainer(JsonNode json) { if (element.has(VALUES_FIELD_NAME)) { if (element.get(VALUES_FIELD_NAME).isArray()) { Iterator elementIter = element.get(VALUES_FIELD_NAME).elements(); - while(elementIter.hasNext()){ + while (elementIter.hasNext()) { values.add(PrimitiveTypeProvider.fromJSON(elementIter.next())); } } else { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/IdQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/IdQueryTermContainer.java index a90aa3a10..eeb717f0e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/IdQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/IdQueryTermContainer.java @@ -2,8 +2,8 @@ public class IdQueryTermContainer extends AbstractQueryTermContainer { - public IdQueryTermContainer(String id){ - setId(id); - } + public IdQueryTermContainer(String id) { + setId(id); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ImageQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ImageQueryTermContainer.java index 986f6662a..b9d8fce1d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ImageQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ImageQueryTermContainer.java @@ -1,130 +1,129 @@ package org.vitrivr.cineast.core.data.query.containers; import georegression.struct.point.Point2D_F32; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.frames.VideoDescriptor; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; import org.vitrivr.cineast.core.util.web.ImageParser; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class ImageQueryTermContainer extends AbstractQueryTermContainer { - private MultiImage img; - private VideoFrame videoFrame; - private ArrayList subitem = new ArrayList(1); - private List>> paths = new ArrayList>>(); - private List>> bgPaths = new ArrayList>>(); - private float relativeStart = 0, relativeEnd = 0; - - /** - * Constructs an {@link ImageQueryTermContainer} from base 64 encoded image data. - * - * @param data The image data that should be converted. - * @param factory The {@link CachedDataFactory} that should be used to generate the {@link MultiImage}. - */ - public ImageQueryTermContainer(String data, CachedDataFactory factory) { - this(ImageParser.dataURLtoBufferedImage(data), factory); - } - - public ImageQueryTermContainer(BufferedImage image, CachedDataFactory factory) { - this.img = factory.newInMemoryMultiImage(image); - } - - public ImageQueryTermContainer(BufferedImage image) { - this(image, CachedDataFactory.getDefault()); - } - - public ImageQueryTermContainer(String data){ - this(data, CachedDataFactory.getDefault()); - } - - public ImageQueryTermContainer(MultiImage img){ - this.img = img; - } - - @Override - public MultiImage getAvgImg() { - return this.img; - } - - @Override - public MultiImage getMedianImg() { - return this.img; - } - - @Override - public VideoFrame getMostRepresentativeFrame() { - if(this.videoFrame == null){ - int id = (getStart() + getEnd()) /2; - this.videoFrame = new VideoFrame(id, 0,this.img, new VideoDescriptor(25, 40, this.img.getWidth(), this.img.getHeight())); - } - return this.videoFrame; - } - - @Override - public int getStart() { - return 0; - } - - @Override - public int getEnd() { - return 0; - } - - @Override - public List getSubtitleItems() { - return this.subitem; - } - - @Override - public float getRelativeStart() { - return relativeStart; - } - - @Override - public float getRelativeEnd() { - return relativeEnd; - } - - public void setRelativeStart(float relativeStart){ - this.relativeStart = relativeStart; - } - - public void setRelativeEnd(float relativeEnd){ - this.relativeEnd = relativeEnd; - } - - @Override - public List>> getPaths() { - return this.paths; - } - - @Override - public List>> getBgPaths() { - return this.bgPaths; - } - - @Override + private MultiImage img; + private VideoFrame videoFrame; + private ArrayList subitem = new ArrayList(1); + private List>> paths = new ArrayList>>(); + private List>> bgPaths = new ArrayList>>(); + private float relativeStart = 0, relativeEnd = 0; + + /** + * Constructs an {@link ImageQueryTermContainer} from base 64 encoded image data. + * + * @param data The image data that should be converted. + * @param factory The {@link CachedDataFactory} that should be used to generate the {@link MultiImage}. + */ + public ImageQueryTermContainer(String data, CachedDataFactory factory) { + this(ImageParser.dataURLtoBufferedImage(data), factory); + } + + public ImageQueryTermContainer(BufferedImage image, CachedDataFactory factory) { + this.img = factory.newInMemoryMultiImage(image); + } + + public ImageQueryTermContainer(BufferedImage image) { + this(image, CachedDataFactory.getDefault()); + } + + public ImageQueryTermContainer(String data) { + this(data, CachedDataFactory.getDefault()); + } + + public ImageQueryTermContainer(MultiImage img) { + this.img = img; + } + + @Override + public MultiImage getAvgImg() { + return this.img; + } + + @Override + public MultiImage getMedianImg() { + return this.img; + } + + @Override + public VideoFrame getMostRepresentativeFrame() { + if (this.videoFrame == null) { + int id = (getStart() + getEnd()) / 2; + this.videoFrame = new VideoFrame(id, 0, this.img, new VideoDescriptor(25, 40, this.img.getWidth(), this.img.getHeight())); + } + return this.videoFrame; + } + + @Override + public int getStart() { + return 0; + } + + @Override + public int getEnd() { + return 0; + } + + @Override + public List getSubtitleItems() { + return this.subitem; + } + + @Override + public float getRelativeStart() { + return relativeStart; + } + + @Override + public float getRelativeEnd() { + return relativeEnd; + } + + public void setRelativeStart(float relativeStart) { + this.relativeStart = relativeStart; + } + + public void setRelativeEnd(float relativeEnd) { + this.relativeEnd = relativeEnd; + } + + @Override + public List>> getPaths() { + return this.paths; + } + + @Override + public List>> getBgPaths() { + return this.bgPaths; + } + + @Override public List getVideoFrames() { - ArrayList _return = new ArrayList(1); - _return.add(this.videoFrame); - return _return; - } - - - public void addPath(LinkedList path){ - this.paths.add(new Pair>(0, path)); - } - - public void addBgPath(LinkedList path){ - this.bgPaths.add(new Pair>(0, path)); - } + ArrayList _return = new ArrayList(1); + _return.add(this.videoFrame); + return _return; + } + + + public void addPath(LinkedList path) { + this.paths.add(new Pair>(0, path)); + } + + public void addBgPath(LinkedList path) { + this.bgPaths.add(new Pair>(0, path)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/LocationQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/LocationQueryTermContainer.java index a35aee3d4..95a060f35 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/LocationQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/LocationQueryTermContainer.java @@ -2,52 +2,53 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.MoreObjects; +import java.util.Optional; import org.vitrivr.cineast.core.data.GpsData; import org.vitrivr.cineast.core.data.Location; import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; -import java.util.Optional; - public class LocationQueryTermContainer extends AbstractQueryTermContainer { - /** The {@link Location} object contained in this {@link LocationQueryTermContainer}. */ - private final Location location; - - /** - * Constructs an {@link LocationQueryTermContainer} from JSON data. - * - * @param json The JSON data that should be converted. - */ - public LocationQueryTermContainer(String json) { - final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); - final JsonNode jsonNode = jsonProvider.toJsonNode(json); - if (jsonNode != null) { - this.location = GpsData.parseLocationFromJson(jsonNode).orElseThrow(() -> new IllegalArgumentException("The provided JSON data did not contain valid GPS information.")); - } else { - throw new IllegalArgumentException("Failed to parse the provided JSON data."); - } - } - - /** - * Constructs an {@link LocationQueryTermContainer} from a {@link Location} object. - * - * @param location The JSON data that should be converted. - */ - public LocationQueryTermContainer(Location location) { - this.location = location; - } - - public static LocationQueryTermContainer of(Location location) { - return new LocationQueryTermContainer(location); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("location", getLocation()).toString(); - } - - @Override - public Optional getLocation() { - return Optional.of(location); + /** + * The {@link Location} object contained in this {@link LocationQueryTermContainer}. + */ + private final Location location; + + /** + * Constructs an {@link LocationQueryTermContainer} from JSON data. + * + * @param json The JSON data that should be converted. + */ + public LocationQueryTermContainer(String json) { + final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); + final JsonNode jsonNode = jsonProvider.toJsonNode(json); + if (jsonNode != null) { + this.location = GpsData.parseLocationFromJson(jsonNode).orElseThrow(() -> new IllegalArgumentException("The provided JSON data did not contain valid GPS information.")); + } else { + throw new IllegalArgumentException("Failed to parse the provided JSON data."); } + } + + /** + * Constructs an {@link LocationQueryTermContainer} from a {@link Location} object. + * + * @param location The JSON data that should be converted. + */ + public LocationQueryTermContainer(Location location) { + this.location = location; + } + + public static LocationQueryTermContainer of(Location location) { + return new LocationQueryTermContainer(location); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("location", getLocation()).toString(); + } + + @Override + public Optional getLocation() { + return Optional.of(location); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ModelQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ModelQueryTermContainer.java index 80f35c4f9..6cfb6fc8f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ModelQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ModelQueryTermContainer.java @@ -1,103 +1,107 @@ package org.vitrivr.cineast.core.data.query.containers; +import java.awt.image.BufferedImage; +import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.data.raw.CachedDataFactory; import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.util.mesh.MeshTransformUtil; import org.vitrivr.cineast.core.util.web.ImageParser; import org.vitrivr.cineast.core.util.web.MeshParser; -import java.awt.image.BufferedImage; - public class ModelQueryTermContainer extends AbstractQueryTermContainer { - /** Original Mesh as transferred by the client. */ - private final Mesh mesh; - - /** KHL transformed version of the original Mesh. */ - private final Mesh normalizedMesh; - - /** Image containing a 2D sketch of the 3D model in question. */ - private final MultiImage image; - - /** - * Constructs an {@link ModelQueryTermContainer} from base 64 encoded JSON data. The constructor assumes either the JSV4 JSON format - * for Meshes OR a valid image (for 2D sketch to 3D model lookup). - * - * @param data The 3D model data that should be converted. - * @param factory The {@link CachedDataFactory} used to create images. - */ - public ModelQueryTermContainer(String data, CachedDataFactory factory) { - if (MeshParser.isValidThreeJSV4Geometry(data)) { - this.mesh = MeshParser.parseThreeJSV4Geometry(data); - this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); - this.image = MultiImage.EMPTY_MULTIIMAGE; - } else if (ImageParser.isValidImage(data)) { - final BufferedImage img = ImageParser.dataURLtoBufferedImage(data); - this.image = factory.newMultiImage(img); - this.mesh = Mesh.EMPTY; - this.normalizedMesh = Mesh.EMPTY; - } else { - throw new IllegalArgumentException("The provided data could not be converted to a Mesh."); - } - } - - public ModelQueryTermContainer(String data){ - this(data, CachedDataFactory.getDefault()); - } - - /** - * Constructor for {@link ModelQueryTermContainer} with a Mesh. Used for Query-by-Example. - * - * @param mesh Mesh for which to create a {@link ModelQueryTermContainer}. - */ - public ModelQueryTermContainer(Mesh mesh) { - this.mesh = new Mesh(mesh); - this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); - this.image = MultiImage.EMPTY_MULTIIMAGE; - } - - /** - * Constructor for {@link ModelQueryTermContainer} with ab image. Used for Query-by-Sketch (2d sketch to 3d model). - * - * @param image BufferedImage for which to create a {@link ModelQueryTermContainer}. - * @param factory The {@link CachedDataFactory} to create the {@link MultiImage} with. - */ - public ModelQueryTermContainer(BufferedImage image, CachedDataFactory factory) { - this.image = factory.newMultiImage(image); - this.mesh = Mesh.EMPTY; - this.normalizedMesh = Mesh.EMPTY; - } - - /** - * Constructor for {@link ModelQueryTermContainer} constructor with an image (treated as 2D sketch). Used for Query-by-2D-Sketch. - * - * @param image Image for which to create a {@link ModelQueryTermContainer}. - */ - public ModelQueryTermContainer(MultiImage image) { - this.image = image; - this.mesh = Mesh.EMPTY; - this.normalizedMesh = Mesh.EMPTY; - } - - @Override - public Mesh getMesh() { - return this.mesh; - } - - @Override - public Mesh getNormalizedMesh() { - return this.normalizedMesh; - } - - @Override - public MultiImage getAvgImg() { - return this.image; - } - - @Override - public MultiImage getMedianImg() { - return this.image; + /** + * Original Mesh as transferred by the client. + */ + private final Mesh mesh; + + /** + * KHL transformed version of the original Mesh. + */ + private final Mesh normalizedMesh; + + /** + * Image containing a 2D sketch of the 3D model in question. + */ + private final MultiImage image; + + /** + * Constructs an {@link ModelQueryTermContainer} from base 64 encoded JSON data. The constructor assumes either the JSV4 JSON format for Meshes OR a valid image (for 2D sketch to 3D model lookup). + * + * @param data The 3D model data that should be converted. + * @param factory The {@link CachedDataFactory} used to create images. + */ + public ModelQueryTermContainer(String data, CachedDataFactory factory) { + if (MeshParser.isValidThreeJSV4Geometry(data)) { + this.mesh = MeshParser.parseThreeJSV4Geometry(data); + this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); + this.image = MultiImage.EMPTY_MULTIIMAGE; + } else if (ImageParser.isValidImage(data)) { + final BufferedImage img = ImageParser.dataURLtoBufferedImage(data); + this.image = factory.newMultiImage(img); + this.mesh = Mesh.EMPTY; + this.normalizedMesh = Mesh.EMPTY; + } else { + throw new IllegalArgumentException("The provided data could not be converted to a Mesh."); } + } + + public ModelQueryTermContainer(String data) { + this(data, CachedDataFactory.getDefault()); + } + + /** + * Constructor for {@link ModelQueryTermContainer} with a Mesh. Used for Query-by-Example. + * + * @param mesh Mesh for which to create a {@link ModelQueryTermContainer}. + */ + public ModelQueryTermContainer(Mesh mesh) { + this.mesh = new Mesh(mesh); + this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); + this.image = MultiImage.EMPTY_MULTIIMAGE; + } + + /** + * Constructor for {@link ModelQueryTermContainer} with ab image. Used for Query-by-Sketch (2d sketch to 3d model). + * + * @param image BufferedImage for which to create a {@link ModelQueryTermContainer}. + * @param factory The {@link CachedDataFactory} to create the {@link MultiImage} with. + */ + public ModelQueryTermContainer(BufferedImage image, CachedDataFactory factory) { + this.image = factory.newMultiImage(image); + this.mesh = Mesh.EMPTY; + this.normalizedMesh = Mesh.EMPTY; + } + + /** + * Constructor for {@link ModelQueryTermContainer} constructor with an image (treated as 2D sketch). Used for Query-by-2D-Sketch. + * + * @param image Image for which to create a {@link ModelQueryTermContainer}. + */ + public ModelQueryTermContainer(MultiImage image) { + this.image = image; + this.mesh = Mesh.EMPTY; + this.normalizedMesh = Mesh.EMPTY; + } + + @Override + public Mesh getMesh() { + return this.mesh; + } + + @Override + public Mesh getNormalizedMesh() { + return this.normalizedMesh; + } + + @Override + public MultiImage getAvgImg() { + return this.image; + } + + @Override + public MultiImage getMedianImg() { + return this.image; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/MotionQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/MotionQueryTermContainer.java index 77cff58be..b5d19e1e7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/MotionQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/MotionQueryTermContainer.java @@ -2,95 +2,94 @@ import com.fasterxml.jackson.databind.JsonNode; import georegression.struct.point.Point2D_F32; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.util.web.DataURLParser; - import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.util.web.DataURLParser; //FIXME integer in pair is never used public class MotionQueryTermContainer extends AbstractQueryTermContainer { - private List>> paths = new ArrayList>>(); - private List>> bgPaths = new ArrayList>>(); + private List>> paths = new ArrayList>>(); + private List>> bgPaths = new ArrayList>>(); - public MotionQueryTermContainer(){ - //empty container - } - - /** - * Constructs an {@link MotionQueryTermContainer} from base 64 encoded JSON data. - * - * @param data The 3D model data that should be converted. - */ - public MotionQueryTermContainer(String data) { - this(DataURLParser.dataURLtoJsonNode(data).orElseThrow(() -> new IllegalArgumentException("Failed to parse the provided motion data."))); - } + public MotionQueryTermContainer() { + //empty container + } - /** - * Constructs an {@link MotionQueryTermContainer} from a JsonNode object. - * - * @param jsonNode The JsonNode representing the motion data. - */ - public MotionQueryTermContainer(JsonNode jsonNode) { - final JsonNode foreground = jsonNode.get("foreground"); - ArrayList> list = nodeToList(foreground); - for (LinkedList path : list) { - this.addPath(path); - } + /** + * Constructs an {@link MotionQueryTermContainer} from base 64 encoded JSON data. + * + * @param data The 3D model data that should be converted. + */ + public MotionQueryTermContainer(String data) { + this(DataURLParser.dataURLtoJsonNode(data).orElseThrow(() -> new IllegalArgumentException("Failed to parse the provided motion data."))); + } - final JsonNode background = jsonNode.get("background"); - list = nodeToList(background); - for (LinkedList path : list) { - this.addPath(path); - } + /** + * Constructs an {@link MotionQueryTermContainer} from a JsonNode object. + * + * @param jsonNode The JsonNode representing the motion data. + */ + public MotionQueryTermContainer(JsonNode jsonNode) { + final JsonNode foreground = jsonNode.get("foreground"); + ArrayList> list = nodeToList(foreground); + for (LinkedList path : list) { + this.addPath(path); } - @Override - public List>> getPaths() { - return this.paths; + final JsonNode background = jsonNode.get("background"); + list = nodeToList(background); + for (LinkedList path : list) { + this.addPath(path); } + } - @Override - public List>> getBgPaths() { - return this.bgPaths; - } + @Override + public List>> getPaths() { + return this.paths; + } - public void addPath(LinkedList path) { - this.paths.add(new Pair>(0, path)); - } + @Override + public List>> getBgPaths() { + return this.bgPaths; + } - public void addBgPath(LinkedList path) { - this.bgPaths.add(new Pair>(0, path)); - } + public void addPath(LinkedList path) { + this.paths.add(new Pair>(0, path)); + } - public static AbstractQueryTermContainer fromJson(JsonNode jsonNode) { - return new MotionQueryTermContainer(jsonNode); - } + public void addBgPath(LinkedList path) { + this.bgPaths.add(new Pair>(0, path)); + } - private static ArrayList> nodeToList(JsonNode jsonNode) { - if (jsonNode == null || !jsonNode.isArray()) { - return new ArrayList<>(); - } - ArrayList> _return = new ArrayList<>(jsonNode.size()); - for (final JsonNode list : jsonNode) { - JsonNode path = list.get("path"); - if (path == null || !path.isArray()) { - continue; - } - int size = path.size(); - LinkedList pathList = new LinkedList(); - for (int i = 0; i < size; ++i) { - JsonNode point = path.get(i); - if (point == null) { - continue; - } - pathList.add(new Point2D_F32(point.get("x").floatValue(), point.get("y").floatValue())); - } - _return.add(pathList); + public static AbstractQueryTermContainer fromJson(JsonNode jsonNode) { + return new MotionQueryTermContainer(jsonNode); + } + + private static ArrayList> nodeToList(JsonNode jsonNode) { + if (jsonNode == null || !jsonNode.isArray()) { + return new ArrayList<>(); + } + ArrayList> _return = new ArrayList<>(jsonNode.size()); + for (final JsonNode list : jsonNode) { + JsonNode path = list.get("path"); + if (path == null || !path.isArray()) { + continue; + } + int size = path.size(); + LinkedList pathList = new LinkedList(); + for (int i = 0; i < size; ++i) { + JsonNode point = path.get(i); + if (point == null) { + continue; } - return _return; + pathList.add(new Point2D_F32(point.get("x").floatValue(), point.get("y").floatValue())); + } + _return.add(pathList); } + return _return; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ParameterisedLocationQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ParameterisedLocationQueryTermContainer.java index 67b7c3dd5..eb2e0940d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ParameterisedLocationQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/ParameterisedLocationQueryTermContainer.java @@ -23,7 +23,7 @@ public class ParameterisedLocationQueryTermContainer extends AbstractQueryTermCo public ParameterisedLocationQueryTermContainer(String json) { final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); String converted = json; - if(json != null && json.startsWith("data")){ + if (json != null && json.startsWith("data")) { converted = DataURLParser.dataURLtoString(json, "application/json"); } final ParameterisedLocation ploc = jsonProvider.toObject(converted, @@ -69,6 +69,7 @@ public static class ParameterisedLocation { public GeoPoint geoPoint; public String parameter; + public ParameterisedLocation() { // Empty constructor for de-/seralisation purposes } @@ -78,6 +79,7 @@ public static class GeoPoint { public float latitude; public float longitude; + public GeoPoint() { // Empty constructor for de-/seralisation purposes } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SemanticMapQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SemanticMapQueryTermContainer.java index f0e0f153e..11feb74ae 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SemanticMapQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SemanticMapQueryTermContainer.java @@ -1,78 +1,84 @@ package org.vitrivr.cineast.core.data.query.containers; import com.fasterxml.jackson.databind.JsonNode; -import org.vitrivr.cineast.core.data.SemanticMap; -import org.vitrivr.cineast.core.util.web.DataURLParser; -import org.vitrivr.cineast.core.util.web.ImageParser; - import java.awt.image.BufferedImage; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import org.vitrivr.cineast.core.data.SemanticMap; +import org.vitrivr.cineast.core.util.web.DataURLParser; +import org.vitrivr.cineast.core.util.web.ImageParser; /** * A {@link AbstractQueryTermContainer} for queries using semantic maps. The class expects a JSON of the following form: - * - * { - * image: , - * map: List of objects mapping the class to the color (as used in the image. - * } + *

    + * { image: , map: List of objects mapping the class to the color (as used in the image. } */ public class SemanticMapQueryTermContainer extends AbstractQueryTermContainer { - /** Field name of the image portion of the JSON. */ - private static final String IMAGE_FIELD_NAME = "image"; + /** + * Field name of the image portion of the JSON. + */ + private static final String IMAGE_FIELD_NAME = "image"; - /** Field name of the map portion of the JSON. */ - private static final String MAP_FIELD_NAME = "map"; + /** + * Field name of the map portion of the JSON. + */ + private static final String MAP_FIELD_NAME = "map"; - /** The {@link SemanticMap} that is used internally. */ - private final SemanticMap map; + /** + * The {@link SemanticMap} that is used internally. + */ + private final SemanticMap map; - public SemanticMapQueryTermContainer(SemanticMap map){ - this.map = map; - } + public SemanticMapQueryTermContainer(SemanticMap map) { + this.map = map; + } - /** - * Constructor for {@link SemanticMapQueryTermContainer} - * - * @param data Base64 encoded representation of the JSON. - */ - public SemanticMapQueryTermContainer(String data){ - this(DataURLParser.dataURLtoJsonNode(data).orElseThrow(() -> new IllegalArgumentException("Failed to parse the provided semantic map data."))); - } + /** + * Constructor for {@link SemanticMapQueryTermContainer} + * + * @param data Base64 encoded representation of the JSON. + */ + public SemanticMapQueryTermContainer(String data) { + this(DataURLParser.dataURLtoJsonNode(data).orElseThrow(() -> new IllegalArgumentException("Failed to parse the provided semantic map data."))); + } - /** - * Constructor for {@link SemanticMapQueryTermContainer} - * - * @param jsonNode JsonObject as expected by {@link SemanticMapQueryTermContainer} - */ - public SemanticMapQueryTermContainer(JsonNode jsonNode){ - if (!jsonNode.has(IMAGE_FIELD_NAME)) throw new IllegalArgumentException("The provided data structure does not contain the required field 'image' (semantic map)."); - if (!jsonNode.has(MAP_FIELD_NAME)) throw new IllegalArgumentException("The provided data structure does not contain the required field 'map' (category to color map)."); + /** + * Constructor for {@link SemanticMapQueryTermContainer} + * + * @param jsonNode JsonObject as expected by {@link SemanticMapQueryTermContainer} + */ + public SemanticMapQueryTermContainer(JsonNode jsonNode) { + if (!jsonNode.has(IMAGE_FIELD_NAME)) { + throw new IllegalArgumentException("The provided data structure does not contain the required field 'image' (semantic map)."); + } + if (!jsonNode.has(MAP_FIELD_NAME)) { + throw new IllegalArgumentException("The provided data structure does not contain the required field 'map' (category to color map)."); + } - final BufferedImage image = ImageParser.dataURLtoBufferedImage(jsonNode.get(IMAGE_FIELD_NAME).asText()); - final Map classes = new LinkedHashMap<>(); + final BufferedImage image = ImageParser.dataURLtoBufferedImage(jsonNode.get(IMAGE_FIELD_NAME).asText()); + final Map classes = new LinkedHashMap<>(); - if (jsonNode.get(MAP_FIELD_NAME).isArray()) { - for (JsonNode node : jsonNode.get(MAP_FIELD_NAME)) { - if (node.isObject()) { - node.fieldNames().forEachRemaining(f -> { - classes.put(f, node.get(f).asText()); - }); - } - } + if (jsonNode.get(MAP_FIELD_NAME).isArray()) { + for (JsonNode node : jsonNode.get(MAP_FIELD_NAME)) { + if (node.isObject()) { + node.fieldNames().forEachRemaining(f -> { + classes.put(f, node.get(f).asText()); + }); } - this.map = new SemanticMap(image, classes); + } } + this.map = new SemanticMap(image, classes); + } - /** - * Getter for {@link SemanticMapQueryTermContainer#map}. - * - * @return {@link SemanticMap} - */ - @Override - public Optional getSemanticMap() { - return Optional.of(this.map); - } + /** + * Getter for {@link SemanticMapQueryTermContainer#map}. + * + * @return {@link SemanticMap} + */ + @Override + public Optional getSemanticMap() { + return Optional.of(this.map); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextQueryTermContainer.java index 73019412a..91612b0f9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextQueryTermContainer.java @@ -2,22 +2,22 @@ public class TextQueryTermContainer extends AbstractQueryTermContainer { - /** - * Text contained in this {@link TextQueryTermContainer}. - */ - private final String text; + /** + * Text contained in this {@link TextQueryTermContainer}. + */ + private final String text; - /** - * Constructs a new {@link TextQueryTermContainer} from the provided text. - * - * @param text Text for which to construct a {@link TextQueryTermContainer}. - */ - public TextQueryTermContainer(String text) { - this.text = (text == null ? "" : text); - } + /** + * Constructs a new {@link TextQueryTermContainer} from the provided text. + * + * @param text Text for which to construct a {@link TextQueryTermContainer}. + */ + public TextQueryTermContainer(String text) { + this.text = (text == null ? "" : text); + } - @Override - public String getText() { - return this.text; - } + @Override + public String getText() { + return this.text; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CacheableData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CacheableData.java index 77bb21df5..b91113dbd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CacheableData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CacheableData.java @@ -4,14 +4,15 @@ /** * A cacheable piece of data as created by a {@link CachedDataFactory}. * - * @version 1.0 + * @version 1.0 * @see CachedDataFactory */ public interface CacheableData { - /** - * Exposes the {@link CachedDataFactory} that created this instance of {@link CacheableData}. - * - * @return {@link CachedDataFactory} reference. - */ - CachedDataFactory factory(); + + /** + * Exposes the {@link CachedDataFactory} that created this instance of {@link CacheableData}. + * + * @return {@link CachedDataFactory} reference. + */ + CachedDataFactory factory(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CachedDataFactory.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CachedDataFactory.java index 8555fe7c0..48283d8a5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CachedDataFactory.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/CachedDataFactory.java @@ -2,14 +2,20 @@ import static java.lang.Thread.MIN_PRIORITY; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.ref.PhantomReference; import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; import java.util.HashSet; import java.util.concurrent.locks.ReentrantLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.data.raw.bytes.ByteData; import org.vitrivr.cineast.core.data.raw.bytes.CachedByteData; @@ -18,21 +24,11 @@ import org.vitrivr.cineast.core.data.raw.images.InMemoryMultiImage; import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; - /** - * This factory class generates {@link ByteData} objects either in memory or backed by a file cache based on a heuristic involving - * the size of the allocated data chunks. - * - * @version 1.1 + * This factory class generates {@link ByteData} objects either in memory or backed by a file cache based on a heuristic involving the size of the allocated data chunks. * + * @version 1.1 * @see ByteData * @see InMemoryByteData * @see CachedByteData @@ -40,289 +36,306 @@ * @see CachedMultiImage */ public class CachedDataFactory { - private static CachedDataFactory defaultInstance = new CachedDataFactory(new CacheConfig()); - - /** Default instance of {@link CachedDataFactory}. */ - public static CachedDataFactory getDefault() { - return defaultInstance; - } - - public static void configureDefault(CacheConfig cacheConfig) { - defaultInstance = new CachedDataFactory(cacheConfig); - } - /** A {@link ReentrantLock} to mediate access to CACHED_REFS. */ - private static final ReentrantLock CACHED_REFS_LOCK = new ReentrantLock(); - - /** Internal set that keeps track of {@link CachedByteData} objects; prevents garbage collection of those references. */ - private static final HashSet> CACHED_REFS = new HashSet<>(); - - /** {@link ReferenceQueue} for {@link CachedByteData} objects. */ - private static final ReferenceQueue CACHED_REF_QUEUE = new ReferenceQueue<>(); - - /** Logger instance used to log errors. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Thread that cleans the cached byte data files. */ - private static final Thread CLEANER_THREAD = new Thread(() -> { - while (true) { - try { - final Reference ref = CACHED_REF_QUEUE.remove(); - - /* Synchronization. */ - CACHED_REFS_LOCK.lock(); - CACHED_REFS.remove(ref); - CACHED_REFS_LOCK.unlock(); - - if (ref instanceof CachedByteDataReference) { - final Path path = ((CachedByteDataReference) ref).path; - try { - Files.deleteIfExists(path); - LOGGER.trace("Temporary cache file was purged: {}", path.toString()); - } catch (IOException e) { - LOGGER.fatal("Failed to delete temporary cache file: {}", path.toString()); - } - } - } catch (InterruptedException e) { - LOGGER.fatal("Cleaner thread was interrupted. Cached objects backed by disk will no longer be purged!"); - } + private static CachedDataFactory defaultInstance = new CachedDataFactory(new CacheConfig()); + + /** + * Default instance of {@link CachedDataFactory}. + */ + public static CachedDataFactory getDefault() { + return defaultInstance; + } + + public static void configureDefault(CacheConfig cacheConfig) { + defaultInstance = new CachedDataFactory(cacheConfig); + } + + /** + * A {@link ReentrantLock} to mediate access to CACHED_REFS. + */ + private static final ReentrantLock CACHED_REFS_LOCK = new ReentrantLock(); + + /** + * Internal set that keeps track of {@link CachedByteData} objects; prevents garbage collection of those references. + */ + private static final HashSet> CACHED_REFS = new HashSet<>(); + + /** + * {@link ReferenceQueue} for {@link CachedByteData} objects. + */ + private static final ReferenceQueue CACHED_REF_QUEUE = new ReferenceQueue<>(); + + /** + * Logger instance used to log errors. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Thread that cleans the cached byte data files. + */ + private static final Thread CLEANER_THREAD = new Thread(() -> { + while (true) { + try { + final Reference ref = CACHED_REF_QUEUE.remove(); + + /* Synchronization. */ + CACHED_REFS_LOCK.lock(); + CACHED_REFS.remove(ref); + CACHED_REFS_LOCK.unlock(); + + if (ref instanceof CachedByteDataReference) { + final Path path = ((CachedByteDataReference) ref).path; + try { + Files.deleteIfExists(path); + LOGGER.trace("Temporary cache file was purged: {}", path.toString()); + } catch (IOException e) { + LOGGER.fatal("Failed to delete temporary cache file: {}", path.toString()); + } } - }); - - static { - CLEANER_THREAD.setName("Cache-File-Cleaner"); - CLEANER_THREAD.setPriority(MIN_PRIORITY); - CLEANER_THREAD.setDaemon(true); - CLEANER_THREAD.start(); + } catch (InterruptedException e) { + LOGGER.fatal("Cleaner thread was interrupted. Cached objects backed by disk will no longer be purged!"); + } } - - /** Reference to {@link CacheConfig} used to setup this {@link CachedDataFactory}. */ - private final CacheConfig config; - - /** Location where this instance of {@link CachedDataFactory} will store its cached images. */ - private final Path cacheLocation; - - /** Keeps track of whether the cache directory has been created yet */ - private boolean cacheDirectoryCreated = false; - - /** - * Inner {@link PhantomReference} implementations that keeps track of the cache path for every {@link CachedByteData}. - */ - private static class CachedByteDataReference extends PhantomReference { - private final Path path; - private CachedByteDataReference(CachedByteData data) { - super(data, CachedDataFactory.CACHED_REF_QUEUE); - CACHED_REFS_LOCK.lock(); - CACHED_REFS.add(this); - CACHED_REFS_LOCK.unlock(); - this.path = data.getPath(); - } - public Path getPath() { - return path; - } + }); + + static { + CLEANER_THREAD.setName("Cache-File-Cleaner"); + CLEANER_THREAD.setPriority(MIN_PRIORITY); + CLEANER_THREAD.setDaemon(true); + CLEANER_THREAD.start(); + } + + /** + * Reference to {@link CacheConfig} used to setup this {@link CachedDataFactory}. + */ + private final CacheConfig config; + + /** + * Location where this instance of {@link CachedDataFactory} will store its cached images. + */ + private final Path cacheLocation; + + /** + * Keeps track of whether the cache directory has been created yet + */ + private boolean cacheDirectoryCreated = false; + + /** + * Inner {@link PhantomReference} implementations that keeps track of the cache path for every {@link CachedByteData}. + */ + private static class CachedByteDataReference extends PhantomReference { + + private final Path path; + + private CachedByteDataReference(CachedByteData data) { + super(data, CachedDataFactory.CACHED_REF_QUEUE); + CACHED_REFS_LOCK.lock(); + CACHED_REFS.add(this); + CACHED_REFS_LOCK.unlock(); + this.path = data.getPath(); } - /** - * Default constructor. - * - * @param config {@link CacheConfig} reference. - */ - public CachedDataFactory(CacheConfig config){ - this.config = config; - this.cacheLocation = this.config.getCacheLocation().resolve("cineast_cache_" + config.getUUID()); + public Path getPath() { + return path; } - - /** Method to lazily create the cache directory exists */ - private void ensureDirectory() { - if (this.cacheDirectoryCreated) { - return; - } - this.cacheDirectoryCreated = true; - try { - Files.createDirectories(this.cacheLocation); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - Files.walk(CachedDataFactory.this.cacheLocation).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } catch (IOException e) { - LOGGER.fatal("Failed to sweep the cache location under {}", this.cacheLocation.toAbsolutePath().toString()); - } - })) ; - } catch (IOException e) { - LOGGER.fatal("Failed to create the cache location under {}", this.cacheLocation.toAbsolutePath().toString()); - } - } - - /** - * Wraps the provided byte array in a {@link ByteData} object and returns it. This method determines whether to use a - * {@link InMemoryByteData} or a {@link CachedByteData} object based on the size of the data object and global - * application settings. - * - * @param data The data that should be wrapped. - * @return {@link ByteData} object. - */ - public ByteData newByteData(byte[] data) { - if (this.config.keepInMemory(data.length)) { - return newInMemoryData(data); - } else { - return newCachedData(data, "default"); - } - } - - /** - * Wraps the provided byte array in a {@link ByteData} object and returns it. This method determines whether to use a - * {@link InMemoryByteData} or a {@link CachedByteData} object based on the size of the data object and global - * application settings. - * - * @param data Size of the data object. - * @param prefix The string prefix used to denote the cache file. - * @return {@link ByteData} object. - */ - public ByteData newByteData(byte[] data, String prefix) { - if (this.config.keepInMemory(data.length)) { - return newInMemoryData(data); - } else { - return newCachedData(data, prefix); - } - } - - /** - * Wraps the provided byte array in a {@link InMemoryByteData} object and returns it. - * - * @param data The data that should be wrapped. - * @return {@link ByteData} object. - */ - public ByteData newInMemoryData(byte[] data) { - return new InMemoryByteData(data, this); + } + + /** + * Default constructor. + * + * @param config {@link CacheConfig} reference. + */ + public CachedDataFactory(CacheConfig config) { + this.config = config; + this.cacheLocation = this.config.getCacheLocation().resolve("cineast_cache_" + config.getUUID()); + } + + /** + * Method to lazily create the cache directory exists + */ + private void ensureDirectory() { + if (this.cacheDirectoryCreated) { + return; } - - /** - * Wraps the provided byte array in a {@link CachedByteData} object and returns it. If for some reason, allocation - * of the {@link CachedByteData} fails, an {@link InMemoryByteData} will be returned instead. - * - * @param data The data that should be wrapped. - * @return {@link ByteData} object. - * @throws UncheckedIOException If the allocation of the {@link CachedByteData} fails and FORCE_DISK_CACHE cache policy is used. - */ - public ByteData newCachedData(byte[] data, String prefix) { - ensureDirectory(); - final CacheConfig.Policy cachePolicy = this.config.getCachingPolicy(); - final Path cacheLocation = this.config.getCacheLocation(); + this.cacheDirectoryCreated = true; + try { + Files.createDirectories(this.cacheLocation); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { - return new CachedByteData(data, Files.createTempFile(cacheLocation, prefix, ".tmp"), this); + Files.walk(CachedDataFactory.this.cacheLocation).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); } catch (IOException e) { - LOGGER.warn("Failed to instantiate an object of type CachedByteDate. Fallback to InMemoryByteData instead."); - return new InMemoryByteData(data, this); + LOGGER.fatal("Failed to sweep the cache location under {}", this.cacheLocation.toAbsolutePath().toString()); } + })); + } catch (IOException e) { + LOGGER.fatal("Failed to create the cache location under {}", this.cacheLocation.toAbsolutePath().toString()); } - - /** - * Creates a new {@link MultiImage} from a {@link BufferedImage}. - * - * @param bimg The {@link BufferedImage} to created the {@link MultiImage} from. - * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. - */ - public MultiImage newMultiImage(BufferedImage bimg) { - return newMultiImage(bimg, null); + } + + /** + * Wraps the provided byte array in a {@link ByteData} object and returns it. This method determines whether to use a {@link InMemoryByteData} or a {@link CachedByteData} object based on the size of the data object and global application settings. + * + * @param data The data that should be wrapped. + * @return {@link ByteData} object. + */ + public ByteData newByteData(byte[] data) { + if (this.config.keepInMemory(data.length)) { + return newInMemoryData(data); + } else { + return newCachedData(data, "default"); } - - /** - * Creates a new {@link MultiImage} from a {@link BufferedImage} and its thumbnail. - * - * @param bimg The {@link BufferedImage} to created the {@link MultiImage} from. - * @param thumb The {@link BufferedImage} representing the thumbnail. - * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. - */ - public MultiImage newMultiImage(BufferedImage bimg, BufferedImage thumb) { - if (this.config.keepInMemory(bimg.getWidth() * bimg.getHeight() * 3 * 8)) { - return new InMemoryMultiImage(bimg, thumb, this); - } else { - return newCachedMultiImage(bimg, thumb, "img"); - } + } + + /** + * Wraps the provided byte array in a {@link ByteData} object and returns it. This method determines whether to use a {@link InMemoryByteData} or a {@link CachedByteData} object based on the size of the data object and global application settings. + * + * @param data Size of the data object. + * @param prefix The string prefix used to denote the cache file. + * @return {@link ByteData} object. + */ + public ByteData newByteData(byte[] data, String prefix) { + if (this.config.keepInMemory(data.length)) { + return newInMemoryData(data); + } else { + return newCachedData(data, prefix); } - - /** - * Creates a new {@link MultiImage} from raw color data. - * - * @param width Width of the image. - * @param height Height of the image. - * @param colors Array of color values. - * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. - */ - public MultiImage newMultiImage(int width, int height, int[] colors) { - height = MultiImage.checkHeight(width, height, colors); - final BufferedImage bimg = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); - bimg.setRGB(0, 0, width, height, colors, 0, width); - if (this.config.keepInMemory(colors.length * 8)) { - return newInMemoryMultiImage(bimg); - } else { - return newCachedMultiImage(bimg, "img"); - } + } + + /** + * Wraps the provided byte array in a {@link InMemoryByteData} object and returns it. + * + * @param data The data that should be wrapped. + * @return {@link ByteData} object. + */ + public ByteData newInMemoryData(byte[] data) { + return new InMemoryByteData(data, this); + } + + /** + * Wraps the provided byte array in a {@link CachedByteData} object and returns it. If for some reason, allocation of the {@link CachedByteData} fails, an {@link InMemoryByteData} will be returned instead. + * + * @param data The data that should be wrapped. + * @return {@link ByteData} object. + * @throws UncheckedIOException If the allocation of the {@link CachedByteData} fails and FORCE_DISK_CACHE cache policy is used. + */ + public ByteData newCachedData(byte[] data, String prefix) { + ensureDirectory(); + final CacheConfig.Policy cachePolicy = this.config.getCachingPolicy(); + final Path cacheLocation = this.config.getCacheLocation(); + try { + return new CachedByteData(data, Files.createTempFile(cacheLocation, prefix, ".tmp"), this); + } catch (IOException e) { + LOGGER.warn("Failed to instantiate an object of type CachedByteDate. Fallback to InMemoryByteData instead."); + return new InMemoryByteData(data, this); } - - - /** - * Creates a new {@link InMemoryMultiImage} from raw color data. - * - * @param width Width of the image. - * @param height Height of the image. - * @param colors Array of color values. - * @return {@link InMemoryMultiImage} - */ - public MultiImage newInMemoryMultiImage(int width, int height, int[] colors) { - height = MultiImage.checkHeight(width, height, colors); - final BufferedImage bimg = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); - bimg.setRGB(0, 0, width, height, colors, 0, width); - return newInMemoryMultiImage(bimg); + } + + /** + * Creates a new {@link MultiImage} from a {@link BufferedImage}. + * + * @param bimg The {@link BufferedImage} to created the {@link MultiImage} from. + * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. + */ + public MultiImage newMultiImage(BufferedImage bimg) { + return newMultiImage(bimg, null); + } + + /** + * Creates a new {@link MultiImage} from a {@link BufferedImage} and its thumbnail. + * + * @param bimg The {@link BufferedImage} to created the {@link MultiImage} from. + * @param thumb The {@link BufferedImage} representing the thumbnail. + * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. + */ + public MultiImage newMultiImage(BufferedImage bimg, BufferedImage thumb) { + if (this.config.keepInMemory(bimg.getWidth() * bimg.getHeight() * 3 * 8)) { + return new InMemoryMultiImage(bimg, thumb, this); + } else { + return newCachedMultiImage(bimg, thumb, "img"); } - - /** - * Wraps the provided {@link BufferedImage} in a {@link InMemoryMultiImage} object and returns it. - * - * @param bimg The {@link BufferedImage} that should be wrapped. - * @return {@link InMemoryMultiImage} - */ - public MultiImage newInMemoryMultiImage(BufferedImage bimg) { - return new InMemoryMultiImage(bimg, this); + } + + /** + * Creates a new {@link MultiImage} from raw color data. + * + * @param width Width of the image. + * @param height Height of the image. + * @param colors Array of color values. + * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, depending on cache settings and memory utilisation. + */ + public MultiImage newMultiImage(int width, int height, int[] colors) { + height = MultiImage.checkHeight(width, height, colors); + final BufferedImage bimg = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + bimg.setRGB(0, 0, width, height, colors, 0, width); + if (this.config.keepInMemory(colors.length * 8)) { + return newInMemoryMultiImage(bimg); + } else { + return newCachedMultiImage(bimg, "img"); } - - /** - * Creates and returns a new {@link CachedMultiImage} from the provided image using the provided cache prefix. If the - * instantiation of the {@link CachedMultiImage} fails, the method is allowed to fallback to a {@link InMemoryMultiImage} - * - * @param image The image from which to create a {@link CachedMultiImage}. - * @param prefix The cache prefix used to name the files. - * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, if former could not be created. - */ - public MultiImage newCachedMultiImage(BufferedImage image, String prefix) { - ensureDirectory(); - try { - final CachedMultiImage cimg = new CachedMultiImage(image, Files.createTempFile(this.cacheLocation, prefix, ".tmp"), this); - new CachedByteDataReference(cimg); /* Enqueue phantom reference for garbage collection. */ - return cimg; - } catch (IOException e) { - LOGGER.warn("Failed to instantiate an object of type CachedMultiImage. Fallback to InMemoryMultiImage instead."); - return new InMemoryMultiImage(image, this); - } + } + + + /** + * Creates a new {@link InMemoryMultiImage} from raw color data. + * + * @param width Width of the image. + * @param height Height of the image. + * @param colors Array of color values. + * @return {@link InMemoryMultiImage} + */ + public MultiImage newInMemoryMultiImage(int width, int height, int[] colors) { + height = MultiImage.checkHeight(width, height, colors); + final BufferedImage bimg = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + bimg.setRGB(0, 0, width, height, colors, 0, width); + return newInMemoryMultiImage(bimg); + } + + /** + * Wraps the provided {@link BufferedImage} in a {@link InMemoryMultiImage} object and returns it. + * + * @param bimg The {@link BufferedImage} that should be wrapped. + * @return {@link InMemoryMultiImage} + */ + public MultiImage newInMemoryMultiImage(BufferedImage bimg) { + return new InMemoryMultiImage(bimg, this); + } + + /** + * Creates and returns a new {@link CachedMultiImage} from the provided image using the provided cache prefix. If the instantiation of the {@link CachedMultiImage} fails, the method is allowed to fallback to a {@link InMemoryMultiImage} + * + * @param image The image from which to create a {@link CachedMultiImage}. + * @param prefix The cache prefix used to name the files. + * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, if former could not be created. + */ + public MultiImage newCachedMultiImage(BufferedImage image, String prefix) { + ensureDirectory(); + try { + final CachedMultiImage cimg = new CachedMultiImage(image, Files.createTempFile(this.cacheLocation, prefix, ".tmp"), this); + new CachedByteDataReference(cimg); /* Enqueue phantom reference for garbage collection. */ + return cimg; + } catch (IOException e) { + LOGGER.warn("Failed to instantiate an object of type CachedMultiImage. Fallback to InMemoryMultiImage instead."); + return new InMemoryMultiImage(image, this); } - - /** - * Creates and returns a new {@link CachedMultiImage} from the provided image using the provided cache prefix. If the - * instantiation of the {@link CachedMultiImage} fails, the method is allowed to fallback to a {@link InMemoryMultiImage} - * - * @param image The image from which to create a {@link CachedMultiImage}. - * @param thumb Pre-computed thumbnail that should be used. - * @param prefix The cache prefix used to name the files. - * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, if former could not be created. - */ - public MultiImage newCachedMultiImage(BufferedImage image, BufferedImage thumb, String prefix) { - ensureDirectory(); - try { - final CachedMultiImage cimg = new CachedMultiImage(image, thumb, Files.createTempFile(this.cacheLocation, prefix, ".tmp"), this); - new CachedByteDataReference(cimg); /* Enqueue phantom reference for garbage collection. */ - return cimg; - } catch (IOException e) { - LOGGER.warn("Failed to instantiate an object of type CachedMultiImage. Fallback to InMemoryMultiImage instead."); - return new InMemoryMultiImage(image, thumb, this); - } + } + + /** + * Creates and returns a new {@link CachedMultiImage} from the provided image using the provided cache prefix. If the instantiation of the {@link CachedMultiImage} fails, the method is allowed to fallback to a {@link InMemoryMultiImage} + * + * @param image The image from which to create a {@link CachedMultiImage}. + * @param thumb Pre-computed thumbnail that should be used. + * @param prefix The cache prefix used to name the files. + * @return {@link CachedMultiImage} or {@link InMemoryMultiImage}, if former could not be created. + */ + public MultiImage newCachedMultiImage(BufferedImage image, BufferedImage thumb, String prefix) { + ensureDirectory(); + try { + final CachedMultiImage cimg = new CachedMultiImage(image, thumb, Files.createTempFile(this.cacheLocation, prefix, ".tmp"), this); + new CachedByteDataReference(cimg); /* Enqueue phantom reference for garbage collection. */ + return cimg; + } catch (IOException e) { + LOGGER.warn("Failed to instantiate an object of type CachedMultiImage. Fallback to InMemoryMultiImage instead."); + return new InMemoryMultiImage(image, thumb, this); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/ByteData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/ByteData.java index 8aa899d99..b0da3ca1a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/ByteData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/ByteData.java @@ -1,37 +1,34 @@ package org.vitrivr.cineast.core.data.raw.bytes; -import org.vitrivr.cineast.core.data.raw.CacheableData; - import java.nio.ByteBuffer; +import org.vitrivr.cineast.core.data.raw.CacheableData; /** - * This interface represents an object that holds arbitrary, immutable data that can be stored in a ByteBuffer. - * The kind of data is implementation specific as are the pattern used to access the data. - * + * This interface represents an object that holds arbitrary, immutable data that can be stored in a ByteBuffer. The kind of data is implementation specific as are the pattern used to access the data. */ public interface ByteData extends CacheableData { - /** - * Returns the size in bytes of this {@link ByteData} packet. - * - * @return Size in bytes. - */ - int size(); - - /** - * Returns a {@link ByteBuffer} that backs this {@link ByteData} object. The {@link ByteBuffer} is supposed to be - * read-only and it's position is supposed to be 0. - * - * @return Read-only {@link ByteBuffer}. - */ - ByteBuffer buffer(); - - /** - * Returns the data in this {@link ByteData} object as byte array. - * - * @return Data of this {@link ByteData} object. - */ - byte[] array(); + + /** + * Returns the size in bytes of this {@link ByteData} packet. + * + * @return Size in bytes. + */ + int size(); + + /** + * Returns a {@link ByteBuffer} that backs this {@link ByteData} object. The {@link ByteBuffer} is supposed to be read-only and it's position is supposed to be 0. + * + * @return Read-only {@link ByteBuffer}. + */ + ByteBuffer buffer(); + + /** + * Returns the data in this {@link ByteData} object as byte array. + * + * @return Data of this {@link ByteData} object. + */ + byte[] array(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/CachedByteData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/CachedByteData.java index 3ad2beb5c..ab0a91061 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/CachedByteData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/CachedByteData.java @@ -1,162 +1,161 @@ package org.vitrivr.cineast.core.data.raw.bytes; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.vitrivr.cineast.core.data.raw.CacheableData; -import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; - import java.io.IOException; import java.io.OutputStream; - import java.lang.ref.SoftReference; - import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.raw.CacheableData; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage; /** - * The {@link CachedByteData} object is an immutable {@link ByteData} object backed by a file cache. The data held by the {@link ByteData} - * object may be garbage collected if memory pressure builds up and must be re-created from the cache when accessed. - * - * A temporary cache file is created upon constructing the {@link CachedByteData} object and holds its content in case the in-memory - * representation gets garbage collected. - * - * @version 1.1 + * The {@link CachedByteData} object is an immutable {@link ByteData} object backed by a file cache. The data held by the {@link ByteData} object may be garbage collected if memory pressure builds up and must be re-created from the cache when accessed. + *

    + * A temporary cache file is created upon constructing the {@link CachedByteData} object and holds its content in case the in-memory representation gets garbage collected. * + * @version 1.1 * @see ByteData * @see CacheableData * @see CachedDataFactory */ public class CachedByteData implements ByteData { - /** Logger instance used to log errors. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. */ - private final CachedDataFactory factory; - - /** The file that backs this {@link CachedByteData} object. */ - protected final Path path; - - /** Size of the {@link CachedByteData}. Because the reference to the underlying data is volatile, this value is stored separately. */ - protected final int size; - - /** ByteBuffer holding the raw data. */ - protected SoftReference data; - - /** - * Constructor for {@link CachedByteData} object. - * - * @param data The byte data with which to initialize the {@link CachedByteData} object - * @param file The path to the file that should is supposed to hold the - * - * @throws IOException If unable to create a cache file. - */ - public CachedByteData(byte[] data, Path file, CachedDataFactory factory) throws IOException { - /* Write data to cache file. */ - try (final OutputStream stream = Files.newOutputStream(file,StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)){ - stream.write(data); - this.path = file; - this.size = data.length; - this.data = new SoftReference<>(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)); - this.factory = factory; - } catch (IOException e) { - LOGGER.error("Failed to write data to cache at {}", file); - LOGGER.error(e); - throw e; - } - } - /** - * Returns the size in bytes of this {@link InMemoryByteData} packet. - * - * @return Size in bytes. - */ - @Override - public synchronized int size() { - return this.size; + /** + * Logger instance used to log errors. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. + */ + private final CachedDataFactory factory; + + /** + * The file that backs this {@link CachedByteData} object. + */ + protected final Path path; + + /** + * Size of the {@link CachedByteData}. Because the reference to the underlying data is volatile, this value is stored separately. + */ + protected final int size; + + /** + * ByteBuffer holding the raw data. + */ + protected SoftReference data; + + /** + * Constructor for {@link CachedByteData} object. + * + * @param data The byte data with which to initialize the {@link CachedByteData} object + * @param file The path to the file that should is supposed to hold the + * @throws IOException If unable to create a cache file. + */ + public CachedByteData(byte[] data, Path file, CachedDataFactory factory) throws IOException { + /* Write data to cache file. */ + try (final OutputStream stream = Files.newOutputStream(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + stream.write(data); + this.path = file; + this.size = data.length; + this.data = new SoftReference<>(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)); + this.factory = factory; + } catch (IOException e) { + LOGGER.error("Failed to write data to cache at {}", file); + LOGGER.error(e); + throw e; } - - /** - * Getter for {@link Path} to cache file. - * - * @return {@link Path} to cache file. - */ - public Path getPath() { - return this.path; + } + + /** + * Returns the size in bytes of this {@link InMemoryByteData} packet. + * + * @return Size in bytes. + */ + @Override + public synchronized int size() { + return this.size; + } + + /** + * Getter for {@link Path} to cache file. + * + * @return {@link Path} to cache file. + */ + public Path getPath() { + return this.path; + } + + /** + * Returns the {@link ByteBuffer} that backs this {@link InMemoryByteData} object. The returned buffer is readonly and cannot be changed. + * + * @return {@link ByteBuffer} object. + */ + @Override + public ByteBuffer buffer() { + ByteBuffer buffer = this.data.get(); + if (buffer == null) { + buffer = this.resurrect(); } - - /** - * Returns the {@link ByteBuffer} that backs this {@link InMemoryByteData} object. The returned buffer is - * readonly and cannot be changed. - * - * @return {@link ByteBuffer} object. - */ - @Override - public ByteBuffer buffer() { - ByteBuffer buffer = this.data.get(); - if (buffer == null) { - buffer = this.resurrect(); - } - if (buffer == null) { - buffer = ByteBuffer.wrap(new byte[1]); - } - return buffer.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN); + if (buffer == null) { + buffer = ByteBuffer.wrap(new byte[1]); } - - /** - * Returns the data in this {@link InMemoryByteData} object as byte array. Directly accesses the - * underlying byte buffer to do so. - * - * @return ByteData of this {@link InMemoryByteData} object. - */ - @Override - public byte[] array() { - ByteBuffer buffer = this.data.get(); - if (buffer == null) { - buffer = this.resurrect(); - } - if (buffer == null) { - return new byte[0]; - } - return buffer.array(); + return buffer.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN); + } + + /** + * Returns the data in this {@link InMemoryByteData} object as byte array. Directly accesses the underlying byte buffer to do so. + * + * @return ByteData of this {@link InMemoryByteData} object. + */ + @Override + public byte[] array() { + ByteBuffer buffer = this.data.get(); + if (buffer == null) { + buffer = this.resurrect(); } - - /** - * Exposes the {@link CachedDataFactory} that created this instance of {@link CachedByteData}. - * - * @return {@link CachedDataFactory} reference. - */ - @Override - public CachedDataFactory factory() { - return this.factory; + if (buffer == null) { + return new byte[0]; } - - /** - * Reads the content of this {@link CachedByteData} object from the cache file. If the cache file exists and is readable, - * this method guarantees to return the {@link ByteBuffer} object that contains the data in the cache. - * - * When invoking this method, the local soft reference to that data is also refreshed. However, there is no guarantee - * that when invoking any of the other methods defined in the {@link ByteData} interface, that this reference is still around. - * - * @return {@link ByteBuffer} loaded from cache or null, if reading into the {@link ByteBuffer} failed. - */ - protected ByteBuffer resurrect() { - try { - /* Allocate a new Buffer according to the size of the CachedByteData object. */ - final byte[] data = Files.readAllBytes(this.path); - final ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); - this.data = new SoftReference<>(buffer); - - /* Return true.*/ - return buffer; - } catch (IOException e) { - LOGGER.error("Failed to read data from cache at {} due to an exception. The data contained in {} is lost!", this.path, this.toString()); - LOGGER.error(e); - return ByteBuffer.wrap(new byte[0]).order(ByteOrder.LITTLE_ENDIAN); - } + return buffer.array(); + } + + /** + * Exposes the {@link CachedDataFactory} that created this instance of {@link CachedByteData}. + * + * @return {@link CachedDataFactory} reference. + */ + @Override + public CachedDataFactory factory() { + return this.factory; + } + + /** + * Reads the content of this {@link CachedByteData} object from the cache file. If the cache file exists and is readable, this method guarantees to return the {@link ByteBuffer} object that contains the data in the cache. + *

    + * When invoking this method, the local soft reference to that data is also refreshed. However, there is no guarantee that when invoking any of the other methods defined in the {@link ByteData} interface, that this reference is still around. + * + * @return {@link ByteBuffer} loaded from cache or null, if reading into the {@link ByteBuffer} failed. + */ + protected ByteBuffer resurrect() { + try { + /* Allocate a new Buffer according to the size of the CachedByteData object. */ + final byte[] data = Files.readAllBytes(this.path); + final ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + this.data = new SoftReference<>(buffer); + + /* Return true.*/ + return buffer; + } catch (IOException e) { + LOGGER.error("Failed to read data from cache at {} due to an exception. The data contained in {} is lost!", this.path, this.toString()); + LOGGER.error(e); + return ByteBuffer.wrap(new byte[0]).order(ByteOrder.LITTLE_ENDIAN); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/InMemoryByteData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/InMemoryByteData.java index 9be5fa67a..19fb45b0c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/InMemoryByteData.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/bytes/InMemoryByteData.java @@ -1,80 +1,80 @@ package org.vitrivr.cineast.core.data.raw.bytes; -import org.vitrivr.cineast.core.data.raw.CacheableData; -import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; - import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.vitrivr.cineast.core.data.raw.CacheableData; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.CachedMultiImage; /** - * The {@link InMemoryByteData} object is an immutable {@link ByteData} object that holds all its data in-memory. The memory - * will be occupied until the {@link InMemoryByteData} is garbage collected. - * - * @version 1.0 + * The {@link InMemoryByteData} object is an immutable {@link ByteData} object that holds all its data in-memory. The memory will be occupied until the {@link InMemoryByteData} is garbage collected. * + * @version 1.0 * @see ByteData * @see CacheableData * @see CachedDataFactory */ public class InMemoryByteData implements ByteData { - /** Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. */ - private final CachedDataFactory factory; + /** + * Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. + */ + private final CachedDataFactory factory; - /** ByteBuffer holding the raw data. */ - private final ByteBuffer data; + /** + * ByteBuffer holding the raw data. + */ + private final ByteBuffer data; - /** - * Constructor for {@link InMemoryByteData} object. - * - * @param data The byte data with which to initialize the {@link InMemoryByteData} object - */ - public InMemoryByteData(byte[] data, CachedDataFactory factory) { - this.data = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); - this.factory = factory; - } + /** + * Constructor for {@link InMemoryByteData} object. + * + * @param data The byte data with which to initialize the {@link InMemoryByteData} object + */ + public InMemoryByteData(byte[] data, CachedDataFactory factory) { + this.data = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + this.factory = factory; + } - /** - * Returns the size in bytes of this {@link InMemoryByteData} packet. - * - * @return Size in bytes. - */ - @Override - public int size() { - return this.data.capacity(); - } + /** + * Returns the size in bytes of this {@link InMemoryByteData} packet. + * + * @return Size in bytes. + */ + @Override + public int size() { + return this.data.capacity(); + } - /** - * Returns the {@link ByteBuffer} that backs this {@link InMemoryByteData} object. - * - * @return {@link ByteBuffer} object. - */ - @Override - public ByteBuffer buffer() { - final ByteBuffer returnBuffer = this.data.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN); - returnBuffer.position(0); - return returnBuffer; - } + /** + * Returns the {@link ByteBuffer} that backs this {@link InMemoryByteData} object. + * + * @return {@link ByteBuffer} object. + */ + @Override + public ByteBuffer buffer() { + final ByteBuffer returnBuffer = this.data.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN); + returnBuffer.position(0); + return returnBuffer; + } - /** - * Returns the data in this {@link InMemoryByteData} object as byte array. Directly accesses the - * underlying byte buffer to do so. - * - * @return ByteData of this {@link InMemoryByteData} object. - */ - @Override - public byte[] array() { - return this.data.array(); - } + /** + * Returns the data in this {@link InMemoryByteData} object as byte array. Directly accesses the underlying byte buffer to do so. + * + * @return ByteData of this {@link InMemoryByteData} object. + */ + @Override + public byte[] array() { + return this.data.array(); + } - /** - * Exposes the {@link CachedDataFactory} that created this instance of {@link CachedByteData}. - * - * @return {@link CachedDataFactory} reference. - */ - @Override - public CachedDataFactory factory() { - return this.factory; - } + /** + * Exposes the {@link CachedDataFactory} that created this instance of {@link CachedByteData}. + * + * @return {@link CachedDataFactory} reference. + */ + @Override + public CachedDataFactory factory() { + return this.factory; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/CachedMultiImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/CachedMultiImage.java index d72a55abe..f505b793b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/CachedMultiImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/CachedMultiImage.java @@ -1,222 +1,224 @@ package org.vitrivr.cineast.core.data.raw.images; -import org.vitrivr.cineast.core.data.raw.CacheableData; -import org.vitrivr.cineast.core.data.raw.bytes.ByteData; -import org.vitrivr.cineast.core.data.raw.bytes.CachedByteData; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; - import java.awt.image.BufferedImage; import java.io.IOException; import java.lang.ref.SoftReference; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Path; +import org.vitrivr.cineast.core.data.raw.CacheableData; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.bytes.CachedByteData; /** - * The {@link CachedMultiImage} object is an immutable representation of a {@link BufferedImage} backed by a file cache. The data held - * by the {@link CachedMultiImage} object may be garbage collected if memory pressure builds up and must be re-created from the cache when accessed. - * - * A temporary cache file is created upon constructing the {@link CachedMultiImage} object and holds its content in case the in-memory - * representation gets garbage collected. - * - * @version 1.0 + * The {@link CachedMultiImage} object is an immutable representation of a {@link BufferedImage} backed by a file cache. The data held by the {@link CachedMultiImage} object may be garbage collected if memory pressure builds up and must be re-created from the cache when accessed. + *

    + * A temporary cache file is created upon constructing the {@link CachedMultiImage} object and holds its content in case the in-memory representation gets garbage collected. * + * @version 1.0 * @see MultiImage * @see CacheableData * @see CachedDataFactory */ public class CachedMultiImage extends CachedByteData implements MultiImage { - /** The width of the cached {@link MultiImage}. */ - private final int width; - - /** The height of the cached {@link MultiImage}. */ - private final int height; - - /** The height of the cached {@link MultiImage}. */ - private final int type; - - /** Soft reference to the thumbnail image. May be garbage collected under memory pressure. */ - private SoftReference thumb; - - /** Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. */ - private final CachedDataFactory factory; - - /** - * Constructor for {@link CachedMultiImage}. - * - * @param img {@link BufferedImage} to create a {@link CachedMultiImage} from. - * @param cacheFile The cache file in which to store {@link CachedMultiImage}. - * @throws IOException If creation of the cache file failed. - */ - public CachedMultiImage(BufferedImage img, Path cacheFile, CachedDataFactory factory) throws IOException { - this(img, null, cacheFile, factory); + /** + * The width of the cached {@link MultiImage}. + */ + private final int width; + + /** + * The height of the cached {@link MultiImage}. + */ + private final int height; + + /** + * The height of the cached {@link MultiImage}. + */ + private final int type; + + /** + * Soft reference to the thumbnail image. May be garbage collected under memory pressure. + */ + private SoftReference thumb; + + /** + * Reference to the {@link CachedDataFactory} that created this {@link CachedMultiImage}. + */ + private final CachedDataFactory factory; + + /** + * Constructor for {@link CachedMultiImage}. + * + * @param img {@link BufferedImage} to create a {@link CachedMultiImage} from. + * @param cacheFile The cache file in which to store {@link CachedMultiImage}. + * @throws IOException If creation of the cache file failed. + */ + public CachedMultiImage(BufferedImage img, Path cacheFile, CachedDataFactory factory) throws IOException { + this(img, null, cacheFile, factory); + } + + /** + * Constructor for {@link CachedMultiImage}. + * + * @param img {@link BufferedImage} to create a {@link CachedMultiImage} from. + * @param thumb {@link BufferedImage} holding the thumbnail image. + * @param cacheFile The cache file in which to store {@link CachedMultiImage}. + * @throws IOException If creation of the cache file failed. + */ + public CachedMultiImage(BufferedImage img, BufferedImage thumb, Path cacheFile, CachedDataFactory factory) throws IOException { + super(toBytes(img), cacheFile, factory); + this.width = img.getWidth(); + this.height = img.getHeight(); + this.type = img.getType(); + this.factory = factory; + if (thumb != null) { + this.thumb = new SoftReference<>(thumb); + } else { + this.thumb = new SoftReference<>(MultiImage.generateThumb(img)); } - - /** - * Constructor for {@link CachedMultiImage}. - * - * @param img {@link BufferedImage} to create a {@link CachedMultiImage} from. - * @param thumb {@link BufferedImage} holding the thumbnail image. - * @param cacheFile The cache file in which to store {@link CachedMultiImage}. - * @throws IOException If creation of the cache file failed. - */ - public CachedMultiImage(BufferedImage img, BufferedImage thumb, Path cacheFile, CachedDataFactory factory) throws IOException { - super(toBytes(img), cacheFile, factory); - this.width = img.getWidth(); - this.height = img.getHeight(); - this.type = img.getType(); - this.factory = factory; - if (thumb != null) { - this.thumb = new SoftReference<>(thumb); - } else { - this.thumb = new SoftReference<>(MultiImage.generateThumb(img)); - } - } - - /** - * Constructor for {@link CachedMultiImage}. - * - * @param colors The array holding the colors of the original, {@link BufferedImage}. - * @param width Width of the image. - * @param height Height of the image. - * @throws IOException If creation of the cache file failed. - */ - public CachedMultiImage(int[] colors, int width, int height, Path cacheFile, CachedDataFactory factory) throws IOException { - super(toBytes(colors, width, height), cacheFile, factory); - - this.width = width; - this.height = height; - this.type = BufferedImage.TYPE_INT_RGB; - this.factory = factory; - - final BufferedImage bimg = new BufferedImage(this.width, this.height, this.type); - bimg.setRGB(0, 0, this.width, this.height, colors, 0, this.width); - this.thumb = new SoftReference<>(MultiImage.generateThumb(bimg)); + } + + /** + * Constructor for {@link CachedMultiImage}. + * + * @param colors The array holding the colors of the original, {@link BufferedImage}. + * @param width Width of the image. + * @param height Height of the image. + * @throws IOException If creation of the cache file failed. + */ + public CachedMultiImage(int[] colors, int width, int height, Path cacheFile, CachedDataFactory factory) throws IOException { + super(toBytes(colors, width, height), cacheFile, factory); + + this.width = width; + this.height = height; + this.type = BufferedImage.TYPE_INT_RGB; + this.factory = factory; + + final BufferedImage bimg = new BufferedImage(this.width, this.height, this.type); + bimg.setRGB(0, 0, this.width, this.height, colors, 0, this.width); + this.thumb = new SoftReference<>(MultiImage.generateThumb(bimg)); + } + + /** + * Getter for the thumbnail image of this {@link CachedMultiImage}. If the thumbnail image reference does not exist anymore, a new thumbnail image will be created from the raw data. + *

    + * Calling this method will cause the soft reference {@link CachedMultiImage#thumb} to be refreshed! However, there is no guarantee that the reference will still be around when invoking this or any other accessor the next time. + * + * @return The thumbnail image for this {@link CachedMultiImage} + */ + @Override + public BufferedImage getThumbnailImage() { + BufferedImage thumbnail = this.thumb.get(); + if (thumbnail == null) { + thumbnail = MultiImage.generateThumb(this.getBufferedImage()); } - - /** - * Getter for the thumbnail image of this {@link CachedMultiImage}. If the thumbnail image reference does not - * exist anymore, a new thumbnail image will be created from the raw data. - * - * Calling this method will cause the soft reference {@link CachedMultiImage#thumb} to be refreshed! However, there is - * no guarantee that the reference will still be around when invoking this or any other accessor the next time. - * - * @return The thumbnail image for this {@link CachedMultiImage} - */ - @Override - public BufferedImage getThumbnailImage() { - BufferedImage thumbnail = this.thumb.get(); - if (thumbnail == null) { - thumbnail = MultiImage.generateThumb(this.getBufferedImage()); - } - this.thumb = new SoftReference<>(thumbnail); - return thumbnail; + this.thumb = new SoftReference<>(thumbnail); + return thumbnail; + } + + /** + * Getter for the colors array representing this {@link CachedMultiImage}. + * + * @return The thumbnail image for this {@link CachedMultiImage} + */ + @Override + public int[] getColors() { + final ByteBuffer buffer = this.buffer(); + final int[] colors = new int[this.width * this.height]; + for (int i = 0; i < colors.length; i++) { + colors[i] = buffer.getInt(); } - - /** - * Getter for the colors array representing this {@link CachedMultiImage}. - * - * @return The thumbnail image for this {@link CachedMultiImage} - */ - @Override - public int[] getColors() { - final ByteBuffer buffer = this.buffer(); - final int[] colors = new int[this.width * this.height]; - for (int i=0; i thumb; - - /** Reference to the colors array of the image. */ - private int[] colors; - - /** The width of the cached {@link MultiImage}. */ - private final int width; - - /** The height of the cached {@link MultiImage}. */ - private final int height; - - /** The height of the cached {@link MultiImage}. */ - private final int type; - - /** Reference to the {@link CachedDataFactory} that created this {@link InMemoryMultiImage}. */ - private final CachedDataFactory factory; - - /** - * Constructor for {@link InMemoryMultiImage}. - * - * @param img {@link BufferedImage} to create a {@link InMemoryMultiImage} from. - */ - public InMemoryMultiImage(BufferedImage img, CachedDataFactory factory){ - this(img, null, factory); - } - - /** - * Constructor for {@link InMemoryMultiImage}. - * - * @param img {@link BufferedImage} to create a {@link InMemoryMultiImage} from. - * @param thumb {@link BufferedImage} holding the thumbnail image. - */ - public InMemoryMultiImage(BufferedImage img, BufferedImage thumb, CachedDataFactory factory) { - this.colors = img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); - this.width = img.getWidth(); - this.height = img.getHeight(); - this.type = img.getType(); - this.factory = factory; - if (thumb != null) { - this.thumb = new SoftReference<>(thumb); - } else { - this.thumb = new SoftReference<>(MultiImage.generateThumb(img)); - } - } - - /** - * Getter for the {@link BufferedImage} held by this {@link InMemoryMultiImage}. The image is reconstructed from the the - * color array. See {@link InMemoryMultiImage#getColors()} - * - * @return The image held by this {@link InMemoryMultiImage} - */ - @Override - public BufferedImage getBufferedImage() { - final BufferedImage image = new BufferedImage(this.width, this.height, this.type); - image.setRGB(0, 0, this.width, this.height, this.colors, 0, this.width); - return image; + /** + * Thumbnail image. This reference will remain in memory as long as {@link InMemoryMultiImage} does. + */ + private SoftReference thumb; + + /** + * Reference to the colors array of the image. + */ + private int[] colors; + + /** + * The width of the cached {@link MultiImage}. + */ + private final int width; + + /** + * The height of the cached {@link MultiImage}. + */ + private final int height; + + /** + * The height of the cached {@link MultiImage}. + */ + private final int type; + + /** + * Reference to the {@link CachedDataFactory} that created this {@link InMemoryMultiImage}. + */ + private final CachedDataFactory factory; + + /** + * Constructor for {@link InMemoryMultiImage}. + * + * @param img {@link BufferedImage} to create a {@link InMemoryMultiImage} from. + */ + public InMemoryMultiImage(BufferedImage img, CachedDataFactory factory) { + this(img, null, factory); + } + + /** + * Constructor for {@link InMemoryMultiImage}. + * + * @param img {@link BufferedImage} to create a {@link InMemoryMultiImage} from. + * @param thumb {@link BufferedImage} holding the thumbnail image. + */ + public InMemoryMultiImage(BufferedImage img, BufferedImage thumb, CachedDataFactory factory) { + this.colors = img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); + this.width = img.getWidth(); + this.height = img.getHeight(); + this.type = img.getType(); + this.factory = factory; + if (thumb != null) { + this.thumb = new SoftReference<>(thumb); + } else { + this.thumb = new SoftReference<>(MultiImage.generateThumb(img)); } - - /** - * Getter for the thumbnail image of this {@link InMemoryMultiImage}. If the thumbnail image reference does not - * exist anymore, a new thumbnail image will be created from the original image. - * - * @return The thumbnail image for this {@link InMemoryMultiImage} - */ - @Override - public BufferedImage getThumbnailImage() { - BufferedImage thumbnail = this.thumb.get(); - if (thumbnail == null) { - thumbnail = MultiImage.generateThumb(this.getBufferedImage()); - } - this.thumb = new SoftReference<>(thumbnail); - return thumbnail; - } - - /** - * Getter for the colors array representing this {@link InMemoryMultiImage}. - * - * @return The thumbnail image for this {@link InMemoryMultiImage} - */ - @Override - public int[] getColors() { - return this.colors; - } - - /** - * Getter for the colors array representing the thumbnail of this {@link InMemoryMultiImage}. - * - * @return Color array of the thumbnail image. - */ - @Override - public int[] getThumbnailColors() { - final BufferedImage thumb = this.getThumbnailImage(); - return this.getThumbnailImage().getRGB(0, 0, thumb.getWidth(), thumb.getHeight(), null, 0, thumb.getWidth()); - } - - /** - * Getter for width value. - * - * @return Width of the {@link MultiImage} - */ - @Override - public int getWidth() { - return this.width; - } - - /** - * Getter for height value. - * - * @return Width of the {@link MultiImage} - */ - @Override - public int getHeight() { - return this.height; + } + + /** + * Getter for the {@link BufferedImage} held by this {@link InMemoryMultiImage}. The image is reconstructed from the the color array. See {@link InMemoryMultiImage#getColors()} + * + * @return The image held by this {@link InMemoryMultiImage} + */ + @Override + public BufferedImage getBufferedImage() { + final BufferedImage image = new BufferedImage(this.width, this.height, this.type); + image.setRGB(0, 0, this.width, this.height, this.colors, 0, this.width); + return image; + } + + /** + * Getter for the thumbnail image of this {@link InMemoryMultiImage}. If the thumbnail image reference does not exist anymore, a new thumbnail image will be created from the original image. + * + * @return The thumbnail image for this {@link InMemoryMultiImage} + */ + @Override + public BufferedImage getThumbnailImage() { + BufferedImage thumbnail = this.thumb.get(); + if (thumbnail == null) { + thumbnail = MultiImage.generateThumb(this.getBufferedImage()); } - - /** - * Getter for this {@link CachedDataFactory}. - * - * @return Factory that created this {@link InMemoryMultiImage} - */ - @Override - public CachedDataFactory factory() { - return this.factory; - } - - @Override - public void clear() {} + this.thumb = new SoftReference<>(thumbnail); + return thumbnail; + } + + /** + * Getter for the colors array representing this {@link InMemoryMultiImage}. + * + * @return The thumbnail image for this {@link InMemoryMultiImage} + */ + @Override + public int[] getColors() { + return this.colors; + } + + /** + * Getter for the colors array representing the thumbnail of this {@link InMemoryMultiImage}. + * + * @return Color array of the thumbnail image. + */ + @Override + public int[] getThumbnailColors() { + final BufferedImage thumb = this.getThumbnailImage(); + return this.getThumbnailImage().getRGB(0, 0, thumb.getWidth(), thumb.getHeight(), null, 0, thumb.getWidth()); + } + + /** + * Getter for width value. + * + * @return Width of the {@link MultiImage} + */ + @Override + public int getWidth() { + return this.width; + } + + /** + * Getter for height value. + * + * @return Width of the {@link MultiImage} + */ + @Override + public int getHeight() { + return this.height; + } + + /** + * Getter for this {@link CachedDataFactory}. + * + * @return Factory that created this {@link InMemoryMultiImage} + */ + @Override + public CachedDataFactory factory() { + return this.factory; + } + + @Override + public void clear() { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java index 7026daed3..d9deb4ea3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/raw/images/MultiImage.java @@ -1,77 +1,78 @@ package org.vitrivr.cineast.core.data.raw.images; +import java.awt.image.BufferedImage; +import java.io.IOException; import net.coobird.thumbnailator.Thumbnails; - import org.vitrivr.cineast.core.data.raw.CacheableData; import org.vitrivr.cineast.core.data.raw.CachedDataFactory; -import java.awt.image.BufferedImage; -import java.io.IOException; - public interface MultiImage extends CacheableData { - + public static final MultiImage EMPTY_MULTIIMAGE = new MultiImage() { - + private int[] emptyArray = new int[0]; private BufferedImage emptyImage = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB); - + @Override public int getWidth() { return 1; } - + @Override public BufferedImage getThumbnailImage() { return emptyImage; } - + @Override public int[] getThumbnailColors() { return emptyArray; } - + @Override public int getHeight() { return 1; } - + @Override public int[] getColors() { return emptyArray; } - + @Override public BufferedImage getBufferedImage() { return emptyImage; } @Override - public CachedDataFactory factory() { return null; } - + public CachedDataFactory factory() { + return null; + } + @Override - public void clear() {} + public void clear() { + } }; - - static final double MAX_THUMB_SIZE = 200; - BufferedImage getBufferedImage(); + static final double MAX_THUMB_SIZE = 200; + + BufferedImage getBufferedImage(); - BufferedImage getThumbnailImage(); + BufferedImage getThumbnailImage(); - int[] getColors(); + int[] getColors(); - int[] getThumbnailColors(); + int[] getThumbnailColors(); - int getWidth(); + int getWidth(); - int getHeight(); + int getHeight(); - CachedDataFactory factory(); + CachedDataFactory factory(); - void clear(); + void clear(); - static int checkHeight(int width, int height, int[] colors){ - if(colors.length / width != height){ + static int checkHeight(int width, int height, int[] colors) { + if (colors.length / width != height) { height = colors.length / width; } return height; @@ -83,11 +84,11 @@ static int checkHeight(int width, int height, int[] colors){ * @param img The image from which to create a thumbnail version. * @return The thumbnail image. */ - static BufferedImage generateThumb(BufferedImage img){ + static BufferedImage generateThumb(BufferedImage img) { double scale = MAX_THUMB_SIZE / Math.max(img.getWidth(), img.getHeight()); - if (scale >= 1 || scale <= 0){ + if (scale >= 1 || scale <= 0) { return img; - } else{ + } else { try { return Thumbnails.of(img).scale(scale).asBufferedImage(); } catch (IOException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/AbstractScoreElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/AbstractScoreElement.java index 184b2715d..4347f9301 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/AbstractScoreElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/AbstractScoreElement.java @@ -3,6 +3,7 @@ import java.util.Objects; abstract class AbstractScoreElement implements ScoreElement { + private final String id; private final double score; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/BooleanSegmentScoreElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/BooleanSegmentScoreElement.java index bcc4720a9..82e03b78e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/BooleanSegmentScoreElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/BooleanSegmentScoreElement.java @@ -4,7 +4,7 @@ public class BooleanSegmentScoreElement implements ScoreElement { private final String id; - public BooleanSegmentScoreElement(String segmentId){ + public BooleanSegmentScoreElement(String segmentId) { this.id = segmentId; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ObjectScoreElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ObjectScoreElement.java index 3c2029136..cfaa5f30a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ObjectScoreElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ObjectScoreElement.java @@ -1,6 +1,7 @@ package org.vitrivr.cineast.core.data.score; public class ObjectScoreElement extends AbstractScoreElement { + public ObjectScoreElement(String objectId, double score) { super(objectId, score); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ScoreElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ScoreElement.java index 175941a0f..75c682bcf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ScoreElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/ScoreElement.java @@ -2,13 +2,10 @@ import com.google.common.collect.ImmutableList; import gnu.trove.map.TObjectDoubleMap; -import gnu.trove.map.hash.TObjectDoubleHashMap; -import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.stream.Stream; import org.vitrivr.cineast.core.data.CorrespondenceFunction; -import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.util.GroupingUtil; /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/SegmentScoreElement.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/SegmentScoreElement.java index 7f8f17a3d..8509df7d5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/SegmentScoreElement.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/score/SegmentScoreElement.java @@ -5,6 +5,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class SegmentScoreElement extends AbstractScoreElement { + public SegmentScoreElement(String segmentId, double score) { super(segmentId, score); } @@ -20,17 +21,19 @@ public String getSegmentId() { /** * Getter for the {@link AbstractQueryTermContainer}'s ID, this {@link SegmentScoreElement} relates to + * * @return The {@link AbstractQueryTermContainer}'s ID, to which this {@link SegmentScoreElement} relates to. */ - public String getQueryContainerId(){ + public String getQueryContainerId() { return queryContainerId; } /** * Setter for the {@link AbstractQueryTermContainer}'s ID, this {@link SegmentScoreElement} relates to + * * @param queryContainerId The ID of the {@link AbstractQueryTermContainer} this {@link SegmentScoreElement} relates to */ - public void setQueryContainerId(String queryContainerId){ + public void setQueryContainerId(String queryContainerId) { this.queryContainerId = queryContainerId; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java index cd736c83e..7f3ece2a6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/AudioSegment.java @@ -1,194 +1,199 @@ package org.vitrivr.cineast.core.data.segments; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - /** - * This AudioSegment is part of the Cineast data model and can hold an arbitrary number of AudioFrames that somehow - * belong together. The class itself is agnostic to how segmenting is organized. - * + * This AudioSegment is part of the Cineast data model and can hold an arbitrary number of AudioFrames that somehow belong together. The class itself is agnostic to how segmenting is organized. + *

    * The AudioSegment implements the SegmentContainer interface and provides access to different, frames-related data. - * + *

    * TODO: Perform basic checks when adding an audio-frame (sample-rate, duration...) */ public class AudioSegment implements SegmentContainer { - /** Segment ID of the AudioSegment. */ - private String segmentId; - - /** ID of the multimedia object this AudioSegment belongs to. */ - private String objectId; - - /** List of AudioFrames in the AudioSegment. */ - private final List frames = new ArrayList<>(); - - /** Total number of samples in the AudioSegment. */ - private int totalSamples; - - /** Total duration of the AudioSegment in seconds. */ - private float totalDuration; - - /** - * The AudioDescriptor that describes the audio in this AudioSegment. It is set by the first frame that is added to the - * segment and must be equal for all the following frames. - */ - private AudioDescriptor descriptor; - - /** - * @return a unique id of this - */ - @Override - public String getId() { - return this.segmentId; - } - - @Override - public void setId(String id) { - this.segmentId = id; - } - @Override - public String getSuperId() { - return this.objectId; + /** + * Segment ID of the AudioSegment. + */ + private String segmentId; + + /** + * ID of the multimedia object this AudioSegment belongs to. + */ + private String objectId; + + /** + * List of AudioFrames in the AudioSegment. + */ + private final List frames = new ArrayList<>(); + + /** + * Total number of samples in the AudioSegment. + */ + private int totalSamples; + + /** + * Total duration of the AudioSegment in seconds. + */ + private float totalDuration; + + /** + * The AudioDescriptor that describes the audio in this AudioSegment. It is set by the first frame that is added to the segment and must be equal for all the following frames. + */ + private AudioDescriptor descriptor; + + /** + * @return a unique id of this + */ + @Override + public String getId() { + return this.segmentId; + } + + @Override + public void setId(String id) { + this.segmentId = id; + } + + @Override + public String getSuperId() { + return this.objectId; + } + + @Override + public void setSuperId(String id) { + this.objectId = id; + } + + /** + * Getter for the list of AudioFrames. + * + * @return Returns an unmodifiable list of audio-frames. + */ + @Override + public List getAudioFrames() { + return Collections.unmodifiableList(this.frames); + } + + /** + * Adds an AudioFrame to the collection of frames and thereby increases both the number of frames and the duration of the segment. + * + * @param frame AudioFrame to add. + * @return boolean True if frame was added, false otherwise. + */ + public boolean addFrame(AudioFrame frame) { + if (frame == null) { + return false; } - - @Override - public void setSuperId(String id) { - this.objectId = id; - } - - /** - * Getter for the list of AudioFrames. - * - * @return Returns an unmodifiable list of audio-frames. - */ - @Override - public List getAudioFrames() { - return Collections.unmodifiableList(this.frames); - } - - /** - * Adds an AudioFrame to the collection of frames and thereby increases both - * the number of frames and the duration of the segment. - * - * @param frame AudioFrame to add. - * @return boolean True if frame was added, false otherwise. - */ - public boolean addFrame(AudioFrame frame) { - if (frame == null) { - return false; - } - if (this.descriptor == null) { - this.descriptor = frame.getDescriptor(); - } - if (!this.descriptor.equals(frame.getDescriptor())) { - return false; - } - - this.totalSamples += frame.numberOfSamples(); - this.totalDuration += frame.getDuration(); - this.frames.add(frame); - - return true; - } - - /** - * Getter for the total number of samples in the AudioSegment. - */ - @Override - public int getNumberOfSamples() { - return this.totalSamples; - } - - /** - * Getter for the total duration of the AudioSegment. - */ - @Override - public float getAudioDuration() { - return totalDuration; - } - - @Override - public float getSamplingrate() { - return this.descriptor.getSamplingrate(); + if (this.descriptor == null) { + this.descriptor = frame.getDescriptor(); } - - @Override - public int getChannels() { - return this.descriptor.getChannels(); + if (!this.descriptor.equals(frame.getDescriptor())) { + return false; } - /** - * Getter for the frame-number of the start-frame. - */ - @Override - public int getStart(){ - if (!this.frames.isEmpty()) { - return (int)this.frames.get(0).getIdx(); - } else { - return 0; - } + this.totalSamples += frame.numberOfSamples(); + this.totalDuration += frame.getDuration(); + this.frames.add(frame); + + return true; + } + + /** + * Getter for the total number of samples in the AudioSegment. + */ + @Override + public int getNumberOfSamples() { + return this.totalSamples; + } + + /** + * Getter for the total duration of the AudioSegment. + */ + @Override + public float getAudioDuration() { + return totalDuration; + } + + @Override + public float getSamplingrate() { + return this.descriptor.getSamplingrate(); + } + + @Override + public int getChannels() { + return this.descriptor.getChannels(); + } + + /** + * Getter for the frame-number of the start-frame. + */ + @Override + public int getStart() { + if (!this.frames.isEmpty()) { + return (int) this.frames.get(0).getIdx(); + } else { + return 0; } - - /** - * Getter for the frame-number of the end-frame. - */ - @Override - public int getEnd(){ - if (!this.frames.isEmpty()) { - return (int)this.frames.get(this.frames.size()-1).getIdx(); - } else { - return 0; - } + } + + /** + * Getter for the frame-number of the end-frame. + */ + @Override + public int getEnd() { + if (!this.frames.isEmpty()) { + return (int) this.frames.get(this.frames.size() - 1).getIdx(); + } else { + return 0; } - - /** - * Getter for the start (in seconds) of this segment. - */ - @Override - public float getAbsoluteStart(){ - if (!this.frames.isEmpty()) { - return this.frames.get(0).getStart(); - } else { - return 0; - } + } + + /** + * Getter for the start (in seconds) of this segment. + */ + @Override + public float getAbsoluteStart() { + if (!this.frames.isEmpty()) { + return this.frames.get(0).getStart(); + } else { + return 0; } - - /** - * Getter for the end (in seconds) of this segment. - */ - @Override - public float getAbsoluteEnd(){ - if (!this.frames.isEmpty()) { - return this.frames.get(this.frames.size()-1).getEnd(); - } else { - return 0; - } + } + + /** + * Getter for the end (in seconds) of this segment. + */ + @Override + public float getAbsoluteEnd() { + if (!this.frames.isEmpty()) { + return this.frames.get(this.frames.size() - 1).getEnd(); + } else { + return 0; } - - /** - * Calculates and returns the Short-term Fourier Transform of the - * current AudioSegment. - * - * @param windowsize Size of the window used during STFT. Must be a power of two. - * @param overlap Overlap in samples between two subsequent windows. - * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. - * @param function WindowFunction to apply before calculating the STFT. - * - * @return STFT of the current AudioSegment or null if the segment is empty. - */ - @Override - public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { - if (2*padding >= windowsize) { - throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); - } - STFT stft = new STFT(windowsize, overlap, padding, function, this.descriptor.getSamplingrate()); - stft.forward(this.getMeanSamplesAsDouble()); - return stft; + } + + /** + * Calculates and returns the Short-term Fourier Transform of the current AudioSegment. + * + * @param windowsize Size of the window used during STFT. Must be a power of two. + * @param overlap Overlap in samples between two subsequent windows. + * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. + * @param function WindowFunction to apply before calculating the STFT. + * @return STFT of the current AudioSegment or null if the segment is empty. + */ + @Override + public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { + if (2 * padding >= windowsize) { + throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); } + STFT stft = new STFT(windowsize, overlap, padding, function, this.descriptor.getSamplingrate()); + stft.forward(this.getMeanSamplesAsDouble()); + return stft; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java index 25a2afe67..bfdcbaccd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/ImageSegment.java @@ -1,90 +1,89 @@ package org.vitrivr.cineast.core.data.segments; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import org.vitrivr.cineast.core.data.frames.VideoDescriptor; -import org.vitrivr.cineast.core.data.frames.VideoFrame; - import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.data.frames.VideoDescriptor; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; public class ImageSegment implements SegmentContainer { - private MultiImage image; - - private VideoFrame videoFrame; - - private String segmentId; - - private String objectId; - - /** - * Default constructor. - * - * @param image The {@link BufferedImage} to create a segment from. - * @param factory The {@link CachedDataFactory} used to convert to a {@link MultiImage}. Handles caching! - */ - public ImageSegment(BufferedImage image, CachedDataFactory factory) { - this.image = factory.newMultiImage(image); - this.videoFrame = new VideoFrame(1, 0, this.image, new VideoDescriptor(25, 40, this.image.getWidth(), this.image.getHeight())); - } - - /** - * @return a unique id of this - */ - @Override - public String getId() { - return this.segmentId; - } - - @Override - public void setId(String id) { - this.segmentId = id; - } - - @Override - public String getSuperId() { - return this.objectId; - } - - @Override - public void setSuperId(String id) { - this.objectId = id; - } - - /** - * Returns the median image, which is the actual image. - */ - @Override - public MultiImage getAvgImg() { - return this.image; - } - - /** - * Returns the average image, which is the actual image. - */ - @Override - public MultiImage getMedianImg() { - return this.image; - } - - /** - * Returns a single frame - the image. - */ - @Override - public List getVideoFrames() { - ArrayList list = new ArrayList<>(1); - list.add(this.videoFrame); - return list; - } - - /** - * Returns a most representative frame - the image. - */ - @Override - public VideoFrame getMostRepresentativeFrame() { - return this.videoFrame; - } + private MultiImage image; + + private VideoFrame videoFrame; + + private String segmentId; + + private String objectId; + + /** + * Default constructor. + * + * @param image The {@link BufferedImage} to create a segment from. + * @param factory The {@link CachedDataFactory} used to convert to a {@link MultiImage}. Handles caching! + */ + public ImageSegment(BufferedImage image, CachedDataFactory factory) { + this.image = factory.newMultiImage(image); + this.videoFrame = new VideoFrame(1, 0, this.image, new VideoDescriptor(25, 40, this.image.getWidth(), this.image.getHeight())); + } + + /** + * @return a unique id of this + */ + @Override + public String getId() { + return this.segmentId; + } + + @Override + public void setId(String id) { + this.segmentId = id; + } + + @Override + public String getSuperId() { + return this.objectId; + } + + @Override + public void setSuperId(String id) { + this.objectId = id; + } + + /** + * Returns the median image, which is the actual image. + */ + @Override + public MultiImage getAvgImg() { + return this.image; + } + + /** + * Returns the average image, which is the actual image. + */ + @Override + public MultiImage getMedianImg() { + return this.image; + } + + /** + * Returns a single frame - the image. + */ + @Override + public List getVideoFrames() { + ArrayList list = new ArrayList<>(1); + list.add(this.videoFrame); + return list; + } + + /** + * Returns a most representative frame - the image. + */ + @Override + public VideoFrame getMostRepresentativeFrame() { + return this.videoFrame; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java index 0771fcfff..8577301fb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/Model3DSegment.java @@ -1,91 +1,106 @@ package org.vitrivr.cineast.core.data.segments; -import org.vitrivr.cineast.core.data.m3d.*; +import org.vitrivr.cineast.core.data.m3d.Mesh; +import org.vitrivr.cineast.core.data.m3d.ReadableMesh; +import org.vitrivr.cineast.core.data.m3d.VoxelGrid; +import org.vitrivr.cineast.core.data.m3d.Voxelizer; +import org.vitrivr.cineast.core.data.m3d.WritableMesh; import org.vitrivr.cineast.core.util.mesh.MeshTransformUtil; public class Model3DSegment implements SegmentContainer { - /** Default Voxelizer instance used for Mesh voxelization. */ - private static final Voxelizer DEFAULT_VOXELIZER = new Voxelizer(0.002f); - - /** Segment ID of the AudioSegment. */ - private String segmentId; - - /** ID of the multimedia object this AudioSegment belongs to. */ - private String objectId; - - /** The original 3D Mesh as extracted from a model file. */ - private final Mesh mesh; - - /** The KHL transformed version of the original Mesh. */ - private final Mesh normalizedMesh; - - /** The 3D VoxelGrid associated with the Model3DSegment. This grid is created lazily. */ - private final Object gridLock = new Object(); - private VoxelGrid grid; - - /** - * Default constructor for Model3DSegment - * - * @param mesh 3D Mesh associated with the segment. - */ - public Model3DSegment(Mesh mesh) { - this.mesh = new Mesh(mesh); - this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); - } - - /** - * @return a unique id of this - */ - @Override - public final String getId() { - return this.segmentId; - } - - @Override - public final void setId(String id) { - this.segmentId = id; - } - - @Override - public final String getSuperId() { - return this.objectId; - } - - @Override - public final void setSuperId(String id) { - this.objectId = id; - } - - /** - * Returns a 3D Mesh associated with this Segment. - * - * @return Mesh - */ - @Override - public final ReadableMesh getMesh() { - return this.mesh; - } - - @Override - public final WritableMesh getNormalizedMesh() { - return this.normalizedMesh; - } - - - /** - * Returns the VoxelGrid associated with this Segment. Calculates the - * grid if needed. - * - * @return VoxelGrid - */ - public final VoxelGrid getGrid() { - synchronized (this.gridLock) { - if (this.grid == null) { - this.grid = DEFAULT_VOXELIZER.voxelize(this.mesh); - } - } - return this.grid; + /** + * Default Voxelizer instance used for Mesh voxelization. + */ + private static final Voxelizer DEFAULT_VOXELIZER = new Voxelizer(0.002f); + + /** + * Segment ID of the AudioSegment. + */ + private String segmentId; + + /** + * ID of the multimedia object this AudioSegment belongs to. + */ + private String objectId; + + /** + * The original 3D Mesh as extracted from a model file. + */ + private final Mesh mesh; + + /** + * The KHL transformed version of the original Mesh. + */ + private final Mesh normalizedMesh; + + /** + * The 3D VoxelGrid associated with the Model3DSegment. This grid is created lazily. + */ + private final Object gridLock = new Object(); + private VoxelGrid grid; + + /** + * Default constructor for Model3DSegment + * + * @param mesh 3D Mesh associated with the segment. + */ + public Model3DSegment(Mesh mesh) { + this.mesh = new Mesh(mesh); + this.normalizedMesh = MeshTransformUtil.khlTransform(mesh, 1.0f); + } + + /** + * @return a unique id of this + */ + @Override + public final String getId() { + return this.segmentId; + } + + @Override + public final void setId(String id) { + this.segmentId = id; + } + + @Override + public final String getSuperId() { + return this.objectId; + } + + @Override + public final void setSuperId(String id) { + this.objectId = id; + } + + /** + * Returns a 3D Mesh associated with this Segment. + * + * @return Mesh + */ + @Override + public final ReadableMesh getMesh() { + return this.mesh; + } + + @Override + public final WritableMesh getNormalizedMesh() { + return this.normalizedMesh; + } + + + /** + * Returns the VoxelGrid associated with this Segment. Calculates the grid if needed. + * + * @return VoxelGrid + */ + public final VoxelGrid getGrid() { + synchronized (this.gridLock) { + if (this.grid == null) { + this.grid = DEFAULT_VOXELIZER.voxelize(this.mesh); + } } + return this.grid; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java index 608368b56..74cfb272a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java @@ -1,30 +1,49 @@ package org.vitrivr.cineast.core.data.segments; -import org.vitrivr.cineast.core.data.providers.*; +import org.vitrivr.cineast.core.data.providers.AudioFrameProvider; +import org.vitrivr.cineast.core.data.providers.AudioSTFTProvider; +import org.vitrivr.cineast.core.data.providers.AvgImgProvider; +import org.vitrivr.cineast.core.data.providers.BooleanExpressionProvider; +import org.vitrivr.cineast.core.data.providers.DurationProvider; +import org.vitrivr.cineast.core.data.providers.FrameListProvider; +import org.vitrivr.cineast.core.data.providers.IdProvider; +import org.vitrivr.cineast.core.data.providers.InstantProvider; +import org.vitrivr.cineast.core.data.providers.LocationProvider; +import org.vitrivr.cineast.core.data.providers.MedianImgProvider; +import org.vitrivr.cineast.core.data.providers.MeshProvider; +import org.vitrivr.cineast.core.data.providers.MostRepresentativeFrameProvider; +import org.vitrivr.cineast.core.data.providers.PathProvider; +import org.vitrivr.cineast.core.data.providers.SemanticMapProvider; +import org.vitrivr.cineast.core.data.providers.SubtitleItemProvider; +import org.vitrivr.cineast.core.data.providers.TagProvider; +import org.vitrivr.cineast.core.data.providers.TextProvider; +import org.vitrivr.cineast.core.data.providers.VoxelGridProvider; import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.features.retriever.Retriever; /** * A {@link SegmentContainer} mainly serves two purposes: - * + *

    * During the offline phase, it is passed to an {@link Extractor}, and during the online phase, it is passed to a {@link Retriever}. */ public interface SegmentContainer extends IdProvider, - AvgImgProvider, - DurationProvider, - MedianImgProvider, - MostRepresentativeFrameProvider, - SubtitleItemProvider, - PathProvider, - TagProvider, - FrameListProvider, - AudioFrameProvider, - AudioSTFTProvider, - MeshProvider, - VoxelGridProvider, - LocationProvider, - InstantProvider, - TextProvider, - SemanticMapProvider, - BooleanExpressionProvider {} + AvgImgProvider, + DurationProvider, + MedianImgProvider, + MostRepresentativeFrameProvider, + SubtitleItemProvider, + PathProvider, + TagProvider, + FrameListProvider, + AudioFrameProvider, + AudioSTFTProvider, + MeshProvider, + VoxelGridProvider, + LocationProvider, + InstantProvider, + TextProvider, + SemanticMapProvider, + BooleanExpressionProvider { + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java index 5a3f4dfff..fc92f05f3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/VideoSegment.java @@ -2,14 +2,18 @@ import boofcv.struct.geo.AssociatedPair; import georegression.struct.point.Point2D_F32; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.data.frames.VideoDescriptor; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.tag.Tag; import org.vitrivr.cineast.core.descriptor.AvgImg; import org.vitrivr.cineast.core.descriptor.MedianImg; @@ -19,385 +23,385 @@ import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - public class VideoSegment implements SegmentContainer { - private static final Logger LOGGER = LogManager.getLogger(); - - private final LinkedList videoFrames = new LinkedList<>(); - private final LinkedList audioFrames = new LinkedList<>(); - private final LinkedList subItems = new LinkedList<>(); - private MultiImage avgImg = null, medianImg = null; - private VideoFrame mostRepresentative = null; - private List>> paths = null; - private List>> bgPaths = null; - private LinkedList>> allPaths = null; - private ArrayList tags = new ArrayList<>(1); - private String movieId; - private String shotId; - - /** Total number of samples in the AudioSegment. */ - private int totalSamples = 0; - - /** Total duration of the AudioSegment in seconds. */ - private float totalAudioDuration = 0.0f; - - /** {@link AudioDescriptor} for the audio stream in this {@link VideoSegment}. Can be null! */ - private AudioDescriptor audioDescriptor = null; - - /** {@link VideoDescriptor} for the video stream in this {@link VideoSegment}. Can be null! */ - private VideoDescriptor videoDescriptor = null; - - /** - * - */ - public VideoSegment() {} - - public VideoSegment(String movieId) { - this.movieId = movieId; + private static final Logger LOGGER = LogManager.getLogger(); + + private final LinkedList videoFrames = new LinkedList<>(); + private final LinkedList audioFrames = new LinkedList<>(); + private final LinkedList subItems = new LinkedList<>(); + private MultiImage avgImg = null, medianImg = null; + private VideoFrame mostRepresentative = null; + private List>> paths = null; + private List>> bgPaths = null; + private LinkedList>> allPaths = null; + private ArrayList tags = new ArrayList<>(1); + private String movieId; + private String shotId; + + /** + * Total number of samples in the AudioSegment. + */ + private int totalSamples = 0; + + /** + * Total duration of the AudioSegment in seconds. + */ + private float totalAudioDuration = 0.0f; + + /** + * {@link AudioDescriptor} for the audio stream in this {@link VideoSegment}. Can be null! + */ + private AudioDescriptor audioDescriptor = null; + + /** + * {@link VideoDescriptor} for the video stream in this {@link VideoSegment}. Can be null! + */ + private VideoDescriptor videoDescriptor = null; + + /** + * + */ + public VideoSegment() { + } + + public VideoSegment(String movieId) { + this.movieId = movieId; + } + + /** + * Returns the number of {@link VideoFrame}s for this {@link VideoSegment}. + * + * @return Number of {@link VideoFrame}s + */ + public int getNumberOfFrames() { + return this.videoFrames.size(); + } + + + /** + * Getter for the list of {@link VideoFrame}s associated with this {@link VideoSegment}. + * + * @return Unmodifiable list of {@link VideoFrame}s + */ + @Override + public List getVideoFrames() { + return Collections.unmodifiableList(this.videoFrames); + } + + /** + * Getter for the list of {@link AudioFrame}s associated with this {@link VideoSegment}. + * + * @return Unmodifiable list of {@link AudioFrame}s + */ + @Override + public List getAudioFrames() { + return Collections.unmodifiableList(this.audioFrames); + } + + /** + * Adds a VideoFrame to the current VideoSegment. If the VideoFrame contains audio, that audio is added too. + * + * @param frame VideoFrame to add to the container. + */ + public boolean addVideoFrame(VideoFrame frame) { + if (frame == null) { + return false; } - - /** - * Returns the number of {@link VideoFrame}s for this {@link VideoSegment}. - * - * @return Number of {@link VideoFrame}s - */ - public int getNumberOfFrames() { - return this.videoFrames.size(); + if (this.videoDescriptor == null) { + this.videoDescriptor = frame.getDescriptor(); } - - - /** - * Getter for the list of {@link VideoFrame}s associated with this {@link VideoSegment}. - * - * @return Unmodifiable list of {@link VideoFrame}s - */ - @Override - public List getVideoFrames() { - return Collections.unmodifiableList(this.videoFrames); + if (!this.videoDescriptor.equals(frame.getDescriptor())) { + return false; } - /** - * Getter for the list of {@link AudioFrame}s associated with this {@link VideoSegment}. - * - * @return Unmodifiable list of {@link AudioFrame}s - */ - @Override - public List getAudioFrames() { - return Collections.unmodifiableList(this.audioFrames); + this.videoFrames.add(frame); + frame.getAudio().ifPresent(this::addAudioFrame); + + /* Add SubtitleItems. */ + frame.getSubtitleItems().forEach(i -> { + if (!this.getSubtitleItems().contains(i)) { + this.getSubtitleItems().add(i); + } + }); + + return true; + } + + /** + * Getter for the total number of audio samples per channel in this {@link VideoSegment}. If the {@link VideoSegment} does not contain any audio, this method returns 0. + * + * @return Number of audio samples + */ + @Override + public int getNumberOfSamples() { + return this.totalSamples; + } + + /** + * Getter for the total audio duration of the {@link VideoSegment}. If the {@link VideoSegment} does not contain any audio, this method returns 0.0f. + * + * @return Duration of audio. + */ + @Override + public float getAudioDuration() { + return this.totalAudioDuration; + } + + /** + * Getter for samplingrate of the audio in this {@link VideoSegment}. If the {@link VideoSegment} does not contain any audio, this method returns 0. + */ + @Override + public float getSamplingrate() { + if (this.audioDescriptor != null) { + return this.audioDescriptor.getSamplingrate(); + } else { + return 0; } - - /** - * Adds a VideoFrame to the current VideoSegment. If the VideoFrame contains - * audio, that audio is added too. - * - * @param frame VideoFrame to add to the container. - */ - public boolean addVideoFrame(VideoFrame frame) { - if (frame == null) { - return false; - } - if (this.videoDescriptor == null) { - this.videoDescriptor = frame.getDescriptor(); - } - if (!this.videoDescriptor.equals(frame.getDescriptor())) { - return false; - } - - this.videoFrames.add(frame); - frame.getAudio().ifPresent(this::addAudioFrame); - - /* Add SubtitleItems. */ - frame.getSubtitleItems().forEach(i -> { - if (!this.getSubtitleItems().contains(i)) { - this.getSubtitleItems().add(i); - } - }); - - return true; + } + + /** + * Getter for number of audio channels for the {@link VideoSegment}. If the {@link VideoSegment} does not contain any audio, this method returns 0. + */ + @Override + public int getChannels() { + if (this.audioDescriptor != null) { + return this.audioDescriptor.getChannels(); + } else { + return 0; } - - /** - * Getter for the total number of audio samples per channel in this {@link VideoSegment}. If the {@link VideoSegment} does - * not contain any audio, this method returns 0. - * - * @return Number of audio samples - */ - @Override - public int getNumberOfSamples() { - return this.totalSamples; + } + + /** + * Calculates and returns the Short-term Fourier Transform of the audio in the current {@link VideoFrame}. If the {@link VideoSegment} does not contain any audio, this method returns null. + * + * @param windowsize Size of the window used during STFT. Must be a power of two. + * @param overlap Overlap in samples between two subsequent windows. + * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. + * @param function WindowFunction to apply before calculating the STFT. + * @return STFT of the audio in the current VideoSegment or null if the segment has no audio. + */ + @Override + public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { + if (this.audioDescriptor == null) { + return null; } - - /** - * Getter for the total audio duration of the {@link VideoSegment}. If the {@link VideoSegment} does not - * contain any audio, this method returns 0.0f. - * - * @return Duration of audio. - */ - @Override - public float getAudioDuration() { - return this.totalAudioDuration; + if (2 * padding >= windowsize) { + throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); } - - /** - * Getter for samplingrate of the audio in this {@link VideoSegment}. If the {@link VideoSegment} does not - * contain any audio, this method returns 0. - */ - @Override - public float getSamplingrate() { - if (this.audioDescriptor != null) { - return this.audioDescriptor.getSamplingrate(); - } else { - return 0; - } + STFT stft = new STFT(windowsize, overlap, padding, function, this.audioDescriptor.getSamplingrate()); + stft.forward(this.getMeanSamplesAsDouble()); + return stft; + } + + private Object getAvgLock = new Object(); + + @Override + public MultiImage getAvgImg() { + synchronized (getAvgLock) { + if (avgImg == null) { + avgImg = AvgImg.getAvg(videoFrames); + } + return avgImg; } + } - /** - * Getter for number of audio channels for the {@link VideoSegment}. If the {@link VideoSegment} does not - * contain any audio, this method returns 0. - */ - @Override - public int getChannels() { - if (this.audioDescriptor != null) { - return this.audioDescriptor.getChannels(); - } else { - return 0; - } - } + private Object getMedianLock = new Object(); - /** - * Calculates and returns the Short-term Fourier Transform of the audio in the current {@link VideoFrame}. If the - * {@link VideoSegment} does not contain any audio, this method returns null. - * - * @param windowsize Size of the window used during STFT. Must be a power of two. - * @param overlap Overlap in samples between two subsequent windows. - * @param padding Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points.. - * @param function WindowFunction to apply before calculating the STFT. - * @return STFT of the audio in the current VideoSegment or null if the segment has no audio. - */ - @Override - public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) { - if (this.audioDescriptor == null) return null; - if (2 * padding >= windowsize) { - throw new IllegalArgumentException("The combined padding must be smaller than the sample window."); - } - STFT stft = new STFT(windowsize, overlap, padding, function, this.audioDescriptor.getSamplingrate()); - stft.forward(this.getMeanSamplesAsDouble()); - return stft; + @Override + public MultiImage getMedianImg() { + synchronized (getMedianLock) { + if (this.medianImg == null) { + this.medianImg = MedianImg.getMedian(videoFrames); + } + return this.medianImg; } - - private Object getAvgLock = new Object(); - - @Override - public MultiImage getAvgImg() { - synchronized (getAvgLock) { - if (avgImg == null) { - avgImg = AvgImg.getAvg(videoFrames); - } - return avgImg; - } + } + + private Object getPathsLock = new Object(); + + @Override + public List>> getPaths() { + synchronized (getPathsLock) { + if (this.paths == null) { + this.allPaths = PathList.getDensePaths(videoFrames); + this.paths = new ArrayList>>(); + this.bgPaths = new ArrayList>>(); + PathList.separateFgBgPaths(videoFrames, this.allPaths, this.paths, this.bgPaths); + } } - - private Object getMedianLock = new Object(); - - @Override - public MultiImage getMedianImg() { - synchronized (getMedianLock) { - if (this.medianImg == null) { - this.medianImg = MedianImg.getMedian(videoFrames); - } - return this.medianImg; - } + return this.paths; + } + + @Override + public List>> getBgPaths() { + synchronized (getPathsLock) { + if (this.bgPaths == null) { + this.allPaths = PathList.getDensePaths(videoFrames); + this.paths = new ArrayList>>(); + this.bgPaths = new ArrayList>>(); + PathList.separateFgBgPaths(videoFrames, this.allPaths, this.paths, this.bgPaths); + } } - - private Object getPathsLock = new Object(); - - @Override - public List>> getPaths() { - synchronized (getPathsLock) { - if (this.paths == null) { - this.allPaths = PathList.getDensePaths(videoFrames); - this.paths = new ArrayList>>(); - this.bgPaths = new ArrayList>>(); - PathList.separateFgBgPaths(videoFrames, this.allPaths, this.paths, this.bgPaths); - } - } - return this.paths; + return this.bgPaths; + } + + /** + * + */ + public void clear() { + LOGGER.trace("clear shot {}", shotId); + for (VideoFrame f : videoFrames) { + f.clear(); } - - @Override - public List>> getBgPaths() { - synchronized (getPathsLock) { - if (this.bgPaths == null) { - this.allPaths = PathList.getDensePaths(videoFrames); - this.paths = new ArrayList>>(); - this.bgPaths = new ArrayList>>(); - PathList.separateFgBgPaths(videoFrames, this.allPaths, this.paths, this.bgPaths); - } - } - return this.bgPaths; + this.videoFrames.clear(); + this.audioFrames.clear(); + this.subItems.clear(); + if (avgImg != null) { + this.avgImg.clear(); + this.avgImg = null; } - - /** - * - */ - public void clear() { - LOGGER.trace("clear shot {}", shotId); - for (VideoFrame f : videoFrames) { - f.clear(); - } - this.videoFrames.clear(); - this.audioFrames.clear(); - this.subItems.clear(); - if (avgImg != null) { - this.avgImg.clear(); - this.avgImg = null; - } - if (medianImg != null) { - this.medianImg.clear(); - this.medianImg = null; - } - if (this.paths != null) { - this.paths.clear(); - this.paths = null; - } - this.mostRepresentative = null; + if (medianImg != null) { + this.medianImg.clear(); + this.medianImg = null; } - - private Object getMostRepresentativeLock = new Object(); - - @Override - public VideoFrame getMostRepresentativeFrame() { - synchronized (getMostRepresentativeLock) { - if (this.mostRepresentative == null) { - this.mostRepresentative = MostRepresentative.getMostRepresentative(this); - } - return this.mostRepresentative; - } + if (this.paths != null) { + this.paths.clear(); + this.paths = null; } - - @Override - public String getId() { - return this.shotId; + this.mostRepresentative = null; + } + + private Object getMostRepresentativeLock = new Object(); + + @Override + public VideoFrame getMostRepresentativeFrame() { + synchronized (getMostRepresentativeLock) { + if (this.mostRepresentative == null) { + this.mostRepresentative = MostRepresentative.getMostRepresentative(this); + } + return this.mostRepresentative; } - - @Override - public String getSuperId() { - return this.movieId; + } + + @Override + public String getId() { + return this.shotId; + } + + @Override + public String getSuperId() { + return this.movieId; + } + + @Override + public void setId(String id) { + this.shotId = id; + } + + @Override + public void setSuperId(String id) { + this.movieId = id; + } + + + @Override + public List getSubtitleItems() { + return this.subItems; + } + + /** + * Returns the frame-number of the first frame in the segment (relative to the entire stream). + */ + @Override + public int getStart() { + if (!this.videoFrames.isEmpty()) { + return this.videoFrames.get(0).getId(); + } else { + return 0; } - - @Override - public void setId(String id) { - this.shotId = id; + } + + /** + * Returns the frame-number of the last frame in the segment (relative to the entire stream). + */ + @Override + public int getEnd() { + if (!this.videoFrames.isEmpty()) { + return this.videoFrames.get(this.videoFrames.size() - 1).getId(); + } else { + return 0; } - - @Override - public void setSuperId(String id) { - this.movieId = id; + } + + /** + * Returns the relative start of the VideoSegment in percent (relative to the entire stream). + */ + @Override + public float getRelativeStart() { + return (1000.0f * this.getStart()) / this.videoDescriptor.getDuration(); + } + + /** + * Returns the relative end of the VideoSegment in percent (relative to the entire stream). + */ + @Override + public float getRelativeEnd() { + return (1000.0f * this.getEnd()) / this.videoDescriptor.getDuration(); + } + + /** + * Returns the absolute start of the VideoSegment in seconds (relative to the entire stream). + */ + @Override + public float getAbsoluteStart() { + if (!this.videoFrames.isEmpty()) { + return this.videoFrames.get(0).getTimestampSeconds(); + } else { + return 0; } - - - @Override - public List getSubtitleItems() { - return this.subItems; + } + + /** + * Returns the absolute end of the VideoSegment in seconds (relative to the entire stream). + */ + @Override + public float getAbsoluteEnd() { + if (!this.videoFrames.isEmpty()) { + return this.videoFrames.get(this.videoFrames.size() - 1).getTimestampSeconds(); + } else { + return 0; } - - /** - * Returns the frame-number of the first frame in the segment (relative to the entire stream). - */ - @Override - public int getStart() { - if (!this.videoFrames.isEmpty()) { - return this.videoFrames.get(0).getId(); - } else { - return 0; - } + } + + @Override + public List getTags() { + return this.tags; + } + + @Override + public String toString() { + return "Shot id: " + this.shotId; + } + + /** + * Adds an {@link AudioFrame} to the collection of {@link AudioFrame}s. + * + * @param frame {@link AudioFrame} to add. + */ + private void addAudioFrame(AudioFrame frame) { + if (frame == null) { + return; } - - /** - * Returns the frame-number of the last frame in the segment (relative to the entire stream). - */ - @Override - public int getEnd() { - if (!this.videoFrames.isEmpty()) { - return this.videoFrames.get(this.videoFrames.size() - 1).getId(); - } else { - return 0; - } - } - - /** - * Returns the relative start of the VideoSegment in percent (relative to the entire stream). - */ - @Override - public float getRelativeStart() { - return (1000.0f * this.getStart()) / this.videoDescriptor.getDuration(); + if (this.audioDescriptor == null) { + this.audioDescriptor = frame.getDescriptor(); } - - /** - * Returns the relative end of the VideoSegment in percent (relative to the entire stream). - */ - @Override - public float getRelativeEnd() { - return (1000.0f * this.getEnd()) / this.videoDescriptor.getDuration(); + if (!this.audioDescriptor.equals(frame.getDescriptor())) { + return; } - /** - * Returns the absolute start of the VideoSegment in seconds (relative to the entire stream). - */ - @Override - public float getAbsoluteStart() { - if (!this.videoFrames.isEmpty()) { - return this.videoFrames.get(0).getTimestampSeconds(); - } else { - return 0; - } - } - - /** - * Returns the absolute end of the VideoSegment in seconds (relative to the entire stream). - */ - @Override - public float getAbsoluteEnd() { - if (!this.videoFrames.isEmpty()) { - return this.videoFrames.get(this.videoFrames.size() - 1).getTimestampSeconds(); - } else { - return 0; - } - } - - @Override - public List getTags() { - return this.tags; - } - - @Override - public String toString() { - return "Shot id: " + this.shotId; - } - - /** - * Adds an {@link AudioFrame} to the collection of {@link AudioFrame}s. - * - * @param frame {@link AudioFrame} to add. - */ - private void addAudioFrame(AudioFrame frame) { - if (frame == null) { - return; - } - if (this.audioDescriptor == null) { - this.audioDescriptor = frame.getDescriptor(); - } - if (!this.audioDescriptor.equals(frame.getDescriptor())) { - return; - } - - this.totalSamples += frame.numberOfSamples(); - this.totalAudioDuration += frame.getDuration(); - this.audioFrames.add(frame); - } + this.totalSamples += frame.numberOfSamples(); + this.totalAudioDuration += frame.getDuration(); + this.audioFrames.add(frame); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteTag.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteTag.java index db47ea2ee..99bc11b48 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteTag.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteTag.java @@ -8,7 +8,9 @@ public class CompleteTag implements Tag { private final String id, name, description; - /** only used in querying */ + /** + * only used in querying + */ private TagPriority priority; public CompleteTag(String id, String name, String description) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteWeightedTag.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteWeightedTag.java index 6ad7686a6..f1d260720 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteWeightedTag.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/CompleteWeightedTag.java @@ -9,7 +9,9 @@ public class CompleteWeightedTag implements WeightedTag { private final String id, name, description; private final float weight; - /** only used in querying */ + /** + * only used in querying + */ private TagPriority priority; public CompleteWeightedTag(String id, String name, String description, float weight) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/IncompleteTag.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/IncompleteTag.java index 377fc5077..672b749b5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/IncompleteTag.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/IncompleteTag.java @@ -11,7 +11,9 @@ public class IncompleteTag implements WeightedTag { private final String id, name, description; private final float weight; - /** only used in querying */ + /** + * only used in querying + */ private TagPriority priority; public IncompleteTag(String id, String name, String description) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/TagPriority.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/TagPriority.java index 42dbdf2ae..b82597212 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/TagPriority.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/tag/TagPriority.java @@ -1,7 +1,7 @@ package org.vitrivr.cineast.core.data.tag; public enum TagPriority { - REQUEST, - REQUIRE, - EXCLUDE + REQUEST, + REQUIRE, + EXCLUDE } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/AbstractPersistencyWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/AbstractPersistencyWriter.java index 2dfee5fff..9c4cedc90 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/AbstractPersistencyWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/AbstractPersistencyWriter.java @@ -8,19 +8,19 @@ public abstract class AbstractPersistencyWriter implements PersistencyWriter { - protected String[] names; + protected String[] names; - protected AbstractPersistencyWriter(String...names){ + protected AbstractPersistencyWriter(String... names) { this.names = names; } - - protected AbstractPersistencyWriter(){ + + protected AbstractPersistencyWriter() { this(GENERIC_ID_COLUMN_QUALIFIER, FEATURE_COLUMN_QUALIFIER); } - + @Override - public void setFieldNames(String...names){ - if(names != null && names.length > 0){ + public void setFieldNames(String... names) { + if (names != null && names.length > 0) { this.names = names; } } @@ -36,10 +36,10 @@ public boolean persist(PersistentTuple tuple) { public boolean idExists(String id) { return exists(GENERIC_ID_COLUMN_QUALIFIER, id); } - + @Override public PersistentTuple generateTuple(Object... objects) { return new PersistentTuple(objects); } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/BooleanExpression.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/BooleanExpression.java index 832c1d33e..6c6ab5324 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/BooleanExpression.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/BooleanExpression.java @@ -1,12 +1,11 @@ package org.vitrivr.cineast.core.db; -import org.apache.commons.lang3.builder.ReflectionToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; public class BooleanExpression { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java index f0f168099..4aa070519 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java @@ -2,6 +2,6 @@ import java.util.function.Supplier; -public interface DBSelectorSupplier extends Supplier{ +public interface DBSelectorSupplier extends Supplier { } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataMessageConverter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataMessageConverter.java index 0fa2375cd..da02a49ad 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataMessageConverter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataMessageConverter.java @@ -1,218 +1,231 @@ package org.vitrivr.cineast.core.db; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import java.util.List; +import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DenseVectorMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.IntVectorMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.SparseVectorMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.VectorMessage; import org.vitrivr.cineast.core.data.FloatArrayIterable; import org.vitrivr.cineast.core.data.IntArrayIterable; -import org.vitrivr.cineast.core.data.providers.primitive.*; +import org.vitrivr.cineast.core.data.providers.primitive.BooleanTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.DoubleTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayProvider; +import org.vitrivr.cineast.core.data.providers.primitive.FloatTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.FloatVectorProvider; +import org.vitrivr.cineast.core.data.providers.primitive.IntArrayProvider; +import org.vitrivr.cineast.core.data.providers.primitive.IntTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.IntVectorProvider; +import org.vitrivr.cineast.core.data.providers.primitive.LongTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.NothingProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +public final class DataMessageConverter { -import java.util.List; + private DataMessageConverter() { + } -public final class DataMessageConverter { + public static final PrimitiveTypeProvider convert(DataMessage message) { + switch (message.getDatatypeCase()) { + case BOOLEANDATA: + return new BooleanTypeProvider(message.getBooleanData()); + case DOUBLEDATA: + return new DoubleTypeProvider(message.getDoubleData()); + case VECTORDATA: + VectorMessage VectorMessage = message.getVectorData(); + switch (VectorMessage.getVectorCase()) { + case DENSEVECTOR: + return new FloatVectorProvider(convert(VectorMessage.getDenseVector())); + case INTVECTOR: + return new IntVectorProvider(convert(VectorMessage.getIntVector())); + case SPARSEVECTOR: + return new FloatVectorProvider(convert(VectorMessage.getSparseVector())); + case VECTOR_NOT_SET: + default: + return new NothingProvider(); + } + case FLOATDATA: + return new FloatTypeProvider(message.getFloatData()); + case INTDATA: + return new IntTypeProvider(message.getIntData()); + case LONGDATA: + return new LongTypeProvider(message.getLongData()); + case STRINGDATA: + return new StringTypeProvider(message.getStringData()); + case DATATYPE_NOT_SET: + default: + return new NothingProvider(); + + } + } + + private static final DataMessage.Builder builder = DataMessage.newBuilder(); + private static final VectorMessage.Builder vectorBuilder = VectorMessage.newBuilder(); + private static final DenseVectorMessage.Builder denseVectorBuilder = DenseVectorMessage.newBuilder(); + private static final IntVectorMessage.Builder intVectorBuilder = IntVectorMessage.newBuilder(); + + public static DataMessage convert(PrimitiveTypeProvider provider) { + switch (provider.getType()) { + case BOOLEAN: + return convert(provider.getBoolean()); + case BYTE: + return convert(provider.getByte()); + case DOUBLE: + return convert(provider.getDouble()); + case FLOAT: + return convert(provider.getFloat()); + case FLOAT_ARRAY: + return convert(provider.getFloatArray()); + case INT: + return convert(provider.getInt()); + case INT_ARRAY: + return convert(provider.getIntArray()); + case LONG: + return convert(provider.getLong()); + case SHORT: + return convert(provider.getShort()); + case STRING: + return convert(provider.getString()); + case UNKNOWN: + default: + throw new IllegalArgumentException("Cannot convert ProviderDataType " + provider.getType() + " to DataMessage"); + + } + } + + + public static final DataMessage convert(boolean bool) { + synchronized (builder) { + builder.clear(); + return builder.setBooleanData(bool).build(); + } + } + + public static final DataMessage convert(int i) { + synchronized (builder) { + builder.clear(); + return builder.setIntData(i).build(); + } + } + + public static DataMessage convert(double d) { + synchronized (builder) { + builder.clear(); + return builder.setDoubleData(d).build(); + } + } + + public static DataMessage convert(float f) { + synchronized (builder) { + builder.clear(); + return builder.setFloatData(f).build(); + } + } + + public static DataMessage convert(long l) { + synchronized (builder) { + builder.clear(); + return builder.setLongData(l).build(); + } + } + + public static DataMessage convert(String s) { + synchronized (builder) { + builder.clear(); + return builder.setStringData(s).build(); + } + } + + public static VectorMessage convertVectorMessage(float[] vector) { + if (vector == null) { + vector = new float[0]; + } + DenseVectorMessage dvmg; + synchronized (denseVectorBuilder) { + dvmg = denseVectorBuilder.clear().addAllVector(new FloatArrayIterable(vector)).build(); + } + synchronized (vectorBuilder) { + vectorBuilder.clear(); + return vectorBuilder.setDenseVector(dvmg).build(); + } + } + + public static DataMessage convert(float[] vector) { + synchronized (builder) { + builder.clear(); + return builder.setVectorData(convertVectorMessage(vector)).build(); + } + } + + public static VectorMessage convertVectorMessage(int[] vector) { + if (vector == null) { + vector = new int[0]; + } + IntVectorMessage ivmg; + synchronized (intVectorBuilder) { + ivmg = intVectorBuilder.clear().addAllVector(new IntArrayIterable(vector)).build(); + } + synchronized (vectorBuilder) { + vectorBuilder.clear(); + return vectorBuilder.setIntVector(ivmg).build(); + } + } + + public static DataMessage convert(int[] vector) { + synchronized (builder) { + builder.clear(); + return builder.setVectorData(convertVectorMessage(vector)).build(); + } + } + + private static float[] convert(DenseVectorMessage message) { + List floatlist = message.getVectorList(); + if (floatlist == null || floatlist.isEmpty()) { + return FloatArrayProvider.DEFAULT_FLOAT_ARRAY; + } + float[] _return = new float[floatlist.size()]; + int i = 0; + for (float f : floatlist) { + _return[i++] = f; + } + return _return; + } + + private static float[] convert(SparseVectorMessage message) { + List indexList = message.getIndexList(); + if (indexList == null || indexList.isEmpty()) { + return FloatArrayProvider.DEFAULT_FLOAT_ARRAY; + } + + int maxIndex = 0; + for (int i : indexList) { + maxIndex = maxIndex < i ? i : maxIndex; + } + + float[] _return = new float[maxIndex + 1]; + + List valueList = message.getDataList(); + + for (int i = 0; i < indexList.size(); ++i) { + _return[indexList.get(i)] = valueList.get(i); + } + + return _return; + } + + private static int[] convert(IntVectorMessage message) { + List intList = message.getVectorList(); + if (intList == null || intList.isEmpty()) { + return IntArrayProvider.DEFAULT_INT_ARRAY; + } + int[] _return = new int[intList.size()]; + int i = 0; + for (int j : intList) { + _return[i++] = j; + } + return _return; + } - private DataMessageConverter(){} - - public static final PrimitiveTypeProvider convert(DataMessage message){ - switch(message.getDatatypeCase()){ - case BOOLEANDATA: - return new BooleanTypeProvider(message.getBooleanData()); - case DOUBLEDATA: - return new DoubleTypeProvider(message.getDoubleData()); - case VECTORDATA: - VectorMessage VectorMessage = message.getVectorData(); - switch(VectorMessage.getVectorCase()){ - case DENSEVECTOR: - return new FloatVectorProvider(convert(VectorMessage.getDenseVector())); - case INTVECTOR: - return new IntVectorProvider(convert(VectorMessage.getIntVector())); - case SPARSEVECTOR: - return new FloatVectorProvider(convert(VectorMessage.getSparseVector())); - case VECTOR_NOT_SET: - default: - return new NothingProvider(); - } - case FLOATDATA: - return new FloatTypeProvider(message.getFloatData()); - case INTDATA: - return new IntTypeProvider(message.getIntData()); - case LONGDATA: - return new LongTypeProvider(message.getLongData()); - case STRINGDATA: - return new StringTypeProvider(message.getStringData()); - case DATATYPE_NOT_SET: - default: - return new NothingProvider(); - - } - } - - private static final DataMessage.Builder builder = DataMessage.newBuilder(); - private static final VectorMessage.Builder vectorBuilder = VectorMessage.newBuilder(); - private static final DenseVectorMessage.Builder denseVectorBuilder = DenseVectorMessage.newBuilder(); - private static final IntVectorMessage.Builder intVectorBuilder = IntVectorMessage.newBuilder(); - - public static DataMessage convert(PrimitiveTypeProvider provider){ - switch(provider.getType()){ - case BOOLEAN: - return convert(provider.getBoolean()); - case BYTE: - return convert(provider.getByte()); - case DOUBLE: - return convert(provider.getDouble()); - case FLOAT: - return convert(provider.getFloat()); - case FLOAT_ARRAY: - return convert(provider.getFloatArray()); - case INT: - return convert(provider.getInt()); - case INT_ARRAY: - return convert(provider.getIntArray()); - case LONG: - return convert(provider.getLong()); - case SHORT: - return convert(provider.getShort()); - case STRING: - return convert(provider.getString()); - case UNKNOWN: - default: - throw new IllegalArgumentException("Cannot convert ProviderDataType " + provider.getType() + " to DataMessage"); - - } - } - - - - public static final DataMessage convert(boolean bool){ - synchronized (builder) { - builder.clear(); - return builder.setBooleanData(bool).build(); - } - } - - public static final DataMessage convert(int i){ - synchronized (builder) { - builder.clear(); - return builder.setIntData(i).build(); - } - } - - public static DataMessage convert(double d) { - synchronized (builder) { - builder.clear(); - return builder.setDoubleData(d).build(); - } - } - - public static DataMessage convert(float f){ - synchronized (builder) { - builder.clear(); - return builder.setFloatData(f).build(); - } - } - - public static DataMessage convert(long l){ - synchronized (builder) { - builder.clear(); - return builder.setLongData(l).build(); - } - } - - public static DataMessage convert(String s){ - synchronized (builder){ - builder.clear(); - return builder.setStringData(s).build(); - } - } - - public static VectorMessage convertVectorMessage(float[] vector){ - if(vector == null){ - vector = new float[0]; - } - DenseVectorMessage dvmg; - synchronized (denseVectorBuilder) { - dvmg = denseVectorBuilder.clear().addAllVector(new FloatArrayIterable(vector)).build(); - } - synchronized (vectorBuilder) { - vectorBuilder.clear(); - return vectorBuilder.setDenseVector(dvmg).build(); - } - } - - public static DataMessage convert(float[] vector){ - synchronized (builder) { - builder.clear(); - return builder.setVectorData(convertVectorMessage(vector)).build(); - } - } - - public static VectorMessage convertVectorMessage(int[] vector){ - if(vector == null){ - vector = new int[0]; - } - IntVectorMessage ivmg; - synchronized (intVectorBuilder) { - ivmg = intVectorBuilder.clear().addAllVector(new IntArrayIterable(vector)).build(); - } - synchronized (vectorBuilder) { - vectorBuilder.clear(); - return vectorBuilder.setIntVector(ivmg).build(); - } - } - - public static DataMessage convert(int[] vector){ - synchronized (builder) { - builder.clear(); - return builder.setVectorData(convertVectorMessage(vector)).build(); - } - } - - private static float[] convert(DenseVectorMessage message){ - List floatlist = message.getVectorList(); - if(floatlist == null || floatlist.isEmpty()){ - return FloatArrayProvider.DEFAULT_FLOAT_ARRAY; - } - float[] _return = new float[floatlist.size()]; - int i = 0; - for(float f : floatlist){ - _return[i++] = f; - } - return _return; - } - - private static float[] convert(SparseVectorMessage message) { - List indexList = message.getIndexList(); - if(indexList == null || indexList.isEmpty()){ - return FloatArrayProvider.DEFAULT_FLOAT_ARRAY; - } - - int maxIndex = 0; - for(int i : indexList){ - maxIndex = maxIndex < i ? i : maxIndex; - } - - float[] _return = new float[maxIndex + 1]; - - List valueList = message.getDataList(); - - for(int i = 0; i < indexList.size(); ++i){ - _return[indexList.get(i)] = valueList.get(i); - } - - return _return; - } - - private static int[] convert(IntVectorMessage message) { - List intList = message.getVectorList(); - if(intList == null || intList.isEmpty()){ - return IntArrayProvider.DEFAULT_INT_ARRAY; - } - int[] _return = new int[intList.size()]; - int i = 0; - for(int j : intList){ - _return[i++] = j; - } - return _return; - } - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/MergeOperation.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/MergeOperation.java index 175a4c020..eb91ea00c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/MergeOperation.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/MergeOperation.java @@ -2,5 +2,5 @@ public enum MergeOperation { - INTERSECT,UNION,EXCEPT + INTERSECT, UNION, EXCEPT } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java index 2b85f766e..486e733ba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java @@ -1,75 +1,73 @@ package org.vitrivr.cineast.core.db; -import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.distance.DistanceElement; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; /** * Helper class to disable database lookups. - * */ public class NoDBSelector implements DBSelector { - @Override - public boolean open(String name) { - return true; - } + @Override + public boolean open(String name) { + return true; + } - @Override - public void close() { - } + @Override + public void close() { + } - @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { - return new ArrayList<>(0); - } + @Override + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + return new ArrayList<>(0); + } @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - return new ArrayList<>(0); - } + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { + return new ArrayList<>(0); + } - @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { - return new ArrayList<>(0); - } + @Override + public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { + return new ArrayList<>(0); + } - @Override - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { - return new ArrayList<>(0); - } + @Override + public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + return new ArrayList<>(0); + } - @Override - public List> getRows(String fieldName, Iterable values) { - return new ArrayList<>(0); - } + @Override + public List> getRows(String fieldName, Iterable values) { + return new ArrayList<>(0); + } - @Override - public List getAll(String column) { - return new ArrayList<>(0); - } + @Override + public List getAll(String column) { + return new ArrayList<>(0); + } - @Override - public List> getAll() { - return new ArrayList<>(0); - } + @Override + public List> getAll() { + return new ArrayList<>(0); + } - @Override - public boolean existsEntity(String name) { - return false; - } + @Override + public boolean existsEntity(String name) { + return false; + } @Override public boolean ping() { return true; } - @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { - return new ArrayList<>(0); - } + @Override + public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + return new ArrayList<>(0); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java index 3e0445556..3649eba41 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java @@ -2,9 +2,8 @@ import java.util.ArrayList; import java.util.List; -import org.vitrivr.cineast.core.db.setup.EntityCreator; - import java.util.function.Supplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; public interface PersistentOperator { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentTuple.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentTuple.java index ee9e93442..1a7e62386 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentTuple.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentTuple.java @@ -5,22 +5,22 @@ public class PersistentTuple { - protected LinkedList elements = new LinkedList<>(); - - protected PersistentTuple(Object...objects){ - if(objects != null){ - for(Object obj : objects){ - addElement(obj); - } - } - } - - public void addElement(Object o){ - this.elements.add(o); - } - - public List getElements(){ - return this.elements; - } - + protected LinkedList elements = new LinkedList<>(); + + protected PersistentTuple(Object... objects) { + if (objects != null) { + for (Object obj : objects) { + addElement(obj); + } + } + } + + public void addElement(Object o) { + this.elements.add(o); + } + + public List getElements() { + return this.elements; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/RelationalOperator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/RelationalOperator.java index 6c4f9bb09..feecf1076 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/RelationalOperator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/RelationalOperator.java @@ -1,41 +1,65 @@ package org.vitrivr.cineast.core.db; public enum RelationalOperator { - /** TRUE if A is equal to B. */ - EQ, + /** + * TRUE if A is equal to B. + */ + EQ, - /** TRUE if A is not equal to B. */ - NEQ, + /** + * TRUE if A is not equal to B. + */ + NEQ, - /** TRUE if A is greater than or equal to B. */ - GEQ, + /** + * TRUE if A is greater than or equal to B. + */ + GEQ, - /** TRUE if A is less than or equal to B. */ - LEQ, + /** + * TRUE if A is less than or equal to B. + */ + LEQ, - /** TRUE if A is greater than B. */ - GREATER, + /** + * TRUE if A is greater than B. + */ + GREATER, - /** TRUE if A is less than B. */ - LESS, + /** + * TRUE if A is less than B. + */ + LESS, - /** TRUE if A is between B and C */ - BETWEEN, + /** + * TRUE if A is between B and C + */ + BETWEEN, - /** TRUE if string A matches string B (SQL LIKE syntax expected). */ - LIKE, + /** + * TRUE if string A matches string B (SQL LIKE syntax expected). + */ + LIKE, - /** TRUE if string A does not match string B (SQL LIKE syntax expected). */ - NLIKE, + /** + * TRUE if string A does not match string B (SQL LIKE syntax expected). + */ + NLIKE, - /** TRUE for fulltext match; Apache Lucene syntax expected. */ - MATCH, + /** + * TRUE for fulltext match; Apache Lucene syntax expected. + */ + MATCH, - /** TRUE if A is null. */ - ISNULL, + /** + * TRUE if A is null. + */ + ISNULL, - /** TRUE if A is not null. */ - ISNOTNULL, + /** + * TRUE if A is not null. + */ + ISNOTNULL, - IN + IN } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java index fca94e356..b6cfa3468 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java @@ -3,6 +3,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage.Code; import org.vitrivr.adampro.grpc.AdamGrpc.AttributeDefinitionMessage; @@ -16,301 +19,291 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.db.setup.EntityDefinition; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; - public class ADAMproEntityCreator implements EntityCreator { - /** - * Wrapper used to send messages to ADAM pro. - */ - private final ADAMproWrapper adampro; - public ADAMproEntityCreator(ADAMproWrapper wrapper){ - this.adampro = wrapper; - } + /** + * Wrapper used to send messages to ADAM pro. + */ + private final ADAMproWrapper adampro; - /** - * Initialises the main entity holding information about multimedia objects in the ADAMpro - * storage engine. - */ - @Override - public boolean createMultiMediaObjectsEntity() { - ArrayList attributes = new ArrayList<>(8); + public ADAMproEntityCreator(ADAMproWrapper wrapper) { + this.adampro = wrapper; + } - AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); + /** + * Initialises the main entity holding information about multimedia objects in the ADAMpro storage engine. + */ + @Override + public boolean createMultiMediaObjectsEntity() { + ArrayList attributes = new ArrayList<>(8); - attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.INT).putAllParams(ImmutableMap.of("indexed", "true")).build()); + AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - builder.clear(); /* Clear builder to erase the indexed flag. */ + attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.INT).putAllParams(ImmutableMap.of("indexed", "true")).build()); - attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).build()); - attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); + builder.clear(); /* Clear builder to erase the indexed flag. */ - CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(MediaObjectDescriptor.ENTITY).addAllAttributes(attributes).build(); + attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).build()); + attributes.add(builder.setName(MediaObjectDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); - AckMessage ack = adampro.createEntityBlocking(message); + CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(MediaObjectDescriptor.ENTITY).addAllAttributes(attributes).build(); - if (ack.getCode() == AckMessage.Code.OK) { - LOGGER.info("Successfully created multimedia object entity."); - } else { - LOGGER.error("Error occurred during creation of multimedia object entity: {}", ack.getMessage()); - } + AckMessage ack = adampro.createEntityBlocking(message); - return ack.getCode() == Code.OK; + if (ack.getCode() == AckMessage.Code.OK) { + LOGGER.info("Successfully created multimedia object entity."); + } else { + LOGGER.error("Error occurred during creation of multimedia object entity: {}", ack.getMessage()); } - @Override - public boolean createMetadataEntity(String tableName) { - final ArrayList fields = new ArrayList<>(4); + return ack.getCode() == Code.OK; + } - final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + @Override + public boolean createMetadataEntity(String tableName) { + final ArrayList fields = new ArrayList<>(4); - builder.clear(); /* Clear builder to erase the indexed flag. */ + final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); + fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); + builder.clear(); /* Clear builder to erase the indexed flag. */ - final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(tableName).addAllAttributes(fields).build(); - final AckMessage ack = adampro.createEntityBlocking(message); + fields.add(builder.setName(MediaObjectMetadataDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); - if (ack.getCode() == AckMessage.Code.OK) { - LOGGER.info("Successfully created metadata entity."); - } else { - LOGGER.error("Error occurred during creation of metadata entity: {}", ack.getMessage()); - } + final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(tableName).addAllAttributes(fields).build(); + final AckMessage ack = adampro.createEntityBlocking(message); - return ack.getCode() == Code.OK; + if (ack.getCode() == AckMessage.Code.OK) { + LOGGER.info("Successfully created metadata entity."); + } else { + LOGGER.error("Error occurred during creation of metadata entity: {}", ack.getMessage()); } - @Override - public boolean createSegmentMetadataEntity(String tableName) { - final ArrayList fields = new ArrayList<>(4); + return ack.getCode() == Code.OK; + } - final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + @Override + public boolean createSegmentMetadataEntity(String tableName) { + final ArrayList fields = new ArrayList<>(4); - builder.clear(); /* Clear builder to erase the indexed flag. */ + final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); + fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); + builder.clear(); /* Clear builder to erase the indexed flag. */ - final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(tableName).addAllAttributes(fields).build(); - final AckMessage ack = adampro.createEntityBlocking(message); + fields.add(builder.setName(MediaSegmentMetadataDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.STRING).build()); - if (ack.getCode() == AckMessage.Code.OK) { - LOGGER.info("Successfully created metadata entity."); - } else { - LOGGER.error("Error occurred during creation of metadata entity: {}", ack.getMessage()); - } + final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(tableName).addAllAttributes(fields).build(); + final AckMessage ack = adampro.createEntityBlocking(message); - return ack.getCode() == Code.OK; + if (ack.getCode() == AckMessage.Code.OK) { + LOGGER.info("Successfully created metadata entity."); + } else { + LOGGER.error("Error occurred during creation of metadata entity: {}", ack.getMessage()); } - /** - * Initialises the entity responsible for holding information about segments of a multimedia object in the - * ADAMpro storage engine. - * - * @see EntityCreator - */ - @Override - public boolean createSegmentEntity() { - final ArrayList fields = new ArrayList<>(6); - - final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - - builder.clear(); /* Clear builder to erase the indexed flag. */ - - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.INT).build()); - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.INT).build()); - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[4]).setAttributetype(AttributeType.INT).build()); - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[5]).setAttributetype(AttributeType.DOUBLE).build()); - fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[6]).setAttributetype(AttributeType.DOUBLE).build()); + return ack.getCode() == Code.OK; + } - final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(MediaSegmentDescriptor.ENTITY).addAllAttributes(fields).build(); + /** + * Initialises the entity responsible for holding information about segments of a multimedia object in the ADAMpro storage engine. + * + * @see EntityCreator + */ + @Override + public boolean createSegmentEntity() { + final ArrayList fields = new ArrayList<>(6); - final AckMessage ack = adampro.createEntityBlocking(message); + final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - if (ack.getCode() == AckMessage.Code.OK) { - LOGGER.info("Successfully created segment entity."); - } else { - LOGGER.error("Error occurred during creation of segment entity: {}", ack.getMessage()); - } + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[0]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[1]).setAttributetype(AttributeType.STRING).putAllParams(ImmutableMap.of("indexed", "true")).build()); - return ack.getCode() == Code.OK; - } - - - - /** - * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field - * called "id", which is of type "string" and has an index. Also, for each of the provided feature attribute a field of the type "vector" - * will be created. - * - * @param featureEntityName ame of the new entity. - * @param unique Whether or not the provided feature should be unique per id. - * @param featureNames List of the feature names. - * @return True on success, false otherwise. - */ - @Override - public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, - String... featureNames) { - final AttributeDefinition[] attributes = Arrays.stream(featureNames) - .map(s -> new AttributeDefinition(s, AttributeDefinition.AttributeType.VECTOR)) - .toArray(AttributeDefinition[]::new); - return this.createFeatureEntity(featureEntityName, unique, attributes); - } - - /** - * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field - * called "id", which is of type "string" and has an index. - * - * @param featureEntityName Name of the new entity. - * @param unique Whether or not the provided feature should be unique per id. - * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. - * @return True on success, false otherwise. - */ - @Override - public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { - final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; - final HashMap hints = new HashMap<>(1); - hints.put("indexed", "true"); - String handler = "parquet"; - for(AttributeDefinition def : attributes){ - if(def.getType().equals(AttributeType.VECTOR) && def.hasHint("handler")){ - handler = def.getHint("handler").get(); - break; - } - } - hints.put("handler", handler); - extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); - System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(featureEntityName, extended); - } + builder.clear(); /* Clear builder to erase the indexed flag. */ - /** - * Creates and initializes an entity with the provided name and the provided attributes. The new entity will have an additional field - * prepended called "id", which is of type "string" and has an index. - * - * @param entityName Name of the new entity. - * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. - * @return True on success, false otherwise. - */ - @Override - public boolean createIdEntity(String entityName, AttributeDefinition... attributes) { - final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; - final HashMap hints = new HashMap<>(1); - hints.put("indexed", "true"); - extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints); - System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(entityName, extended); - } + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[2]).setAttributetype(AttributeType.INT).build()); + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[3]).setAttributetype(AttributeType.INT).build()); + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[4]).setAttributetype(AttributeType.INT).build()); + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[5]).setAttributetype(AttributeType.DOUBLE).build()); + fields.add(builder.setName(MediaSegmentDescriptor.FIELDNAMES[6]).setAttributetype(AttributeType.DOUBLE).build()); - /** - * Creates and initializes an entity with the provided name and the provided attributes. - * - * @param entityName Name of the new entity. - * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. - * @return True on success, false otherwise. - */ - @Override - public boolean createEntity(String entityName, AttributeDefinition... attributes) { - return this.createEntity( - new EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build() - ); - } + final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(MediaSegmentDescriptor.ENTITY).addAllAttributes(fields).build(); - @Override - public boolean createEntity(EntityDefinition def){ - final ArrayList fieldList = new ArrayList<>(); - final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); - - for (AttributeDefinition attribute : def.getAttributes()) { - builder.setName(attribute.getName()).setAttributetype(mapAttributeType(attribute.getType())); - attribute.ifHintPresent("handler", builder::setHandler); - //builder.setHandler("cassandra"); - attribute.ifHintPresent("indexed", h -> builder.putAllParams(ImmutableMap.of("indexed", h))); - fieldList.add(builder.build()); - builder.clear(); - } - - final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(def.getEntityName().toLowerCase()).addAllAttributes(fieldList).build(); - final AckMessage ack = adampro.createEntityBlocking(message); - - if (ack.getCode() == AckMessage.Code.OK) { - LOGGER.info("Successfully created entity '{}'", def.getEntityName()); - } else { - LOGGER.error("Error while creating entity {}: '{}'", def.getEntityName(), ack.getMessage()); - } - - return ack.getCode() == Code.OK; - } + final AckMessage ack = adampro.createEntityBlocking(message); - /* (non-Javadoc) - * @see org.vitrivr.cineast.core.db.setup.IEntityCreator#existsEntity(java.lang.String) - */ - @Override - public boolean existsEntity(String entityName) { - return this.adampro.existsEntityBlocking(entityName); + if (ack.getCode() == AckMessage.Code.OK) { + LOGGER.info("Successfully created segment entity."); + } else { + LOGGER.error("Error occurred during creation of segment entity: {}", ack.getMessage()); } - @Override - public boolean dropEntity(String entityName) { - return this.adampro.dropEntityBlocking(entityName); + return ack.getCode() == Code.OK; + } + + + /** + * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field called "id", which is of type "string" and has an index. Also, for each of the provided feature attribute a field of the type "vector" will be created. + * + * @param featureEntityName ame of the new entity. + * @param unique Whether or not the provided feature should be unique per id. + * @param featureNames List of the feature names. + * @return True on success, false otherwise. + */ + @Override + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, + String... featureNames) { + final AttributeDefinition[] attributes = Arrays.stream(featureNames) + .map(s -> new AttributeDefinition(s, AttributeDefinition.AttributeType.VECTOR)) + .toArray(AttributeDefinition[]::new); + return this.createFeatureEntity(featureEntityName, unique, attributes); + } + + /** + * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field called "id", which is of type "string" and has an index. + * + * @param featureEntityName Name of the new entity. + * @param unique Whether or not the provided feature should be unique per id. + * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. + * @return True on success, false otherwise. + */ + @Override + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { + final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; + final HashMap hints = new HashMap<>(1); + hints.put("indexed", "true"); + String handler = "parquet"; + for (AttributeDefinition def : attributes) { + if (def.getType().equals(AttributeType.VECTOR) && def.hasHint("handler")) { + handler = def.getHint("handler").get(); + break; + } } - - @Override - public boolean createHashNonUniqueIndex(String entityName, String column) { - /* If you are ever going to benchmark adampro vs cottontail, you should implement this method*/ - throw new UnsupportedOperationException(); + hints.put("handler", handler); + extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); + System.arraycopy(attributes, 0, extended, 1, attributes.length); + return this.createEntity(featureEntityName, extended); + } + + /** + * Creates and initializes an entity with the provided name and the provided attributes. The new entity will have an additional field prepended called "id", which is of type "string" and has an index. + * + * @param entityName Name of the new entity. + * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. + * @return True on success, false otherwise. + */ + @Override + public boolean createIdEntity(String entityName, AttributeDefinition... attributes) { + final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; + final HashMap hints = new HashMap<>(1); + hints.put("indexed", "true"); + extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints); + System.arraycopy(attributes, 0, extended, 1, attributes.length); + return this.createEntity(entityName, extended); + } + + /** + * Creates and initializes an entity with the provided name and the provided attributes. + * + * @param entityName Name of the new entity. + * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. + * @return True on success, false otherwise. + */ + @Override + public boolean createEntity(String entityName, AttributeDefinition... attributes) { + return this.createEntity( + new EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build() + ); + } + + @Override + public boolean createEntity(EntityDefinition def) { + final ArrayList fieldList = new ArrayList<>(); + final AttributeDefinitionMessage.Builder builder = AttributeDefinitionMessage.newBuilder(); + + for (AttributeDefinition attribute : def.getAttributes()) { + builder.setName(attribute.getName()).setAttributetype(mapAttributeType(attribute.getType())); + attribute.ifHintPresent("handler", builder::setHandler); + //builder.setHandler("cassandra"); + attribute.ifHintPresent("indexed", h -> builder.putAllParams(ImmutableMap.of("indexed", h))); + fieldList.add(builder.build()); + builder.clear(); } - public boolean dropIndex(String indexName){ - return this.adampro.dropIndexBlocking(indexName); - } + final CreateEntityMessage message = CreateEntityMessage.newBuilder().setEntity(def.getEntityName().toLowerCase()).addAllAttributes(fieldList).build(); + final AckMessage ack = adampro.createEntityBlocking(message); - /* (non-Javadoc) - * @see org.vitrivr.cineast.core.db.setup.IEntityCreator#close() - */ - @Override - public void close() { - //this.adampro.close(); + if (ack.getCode() == AckMessage.Code.OK) { + LOGGER.info("Successfully created entity '{}'", def.getEntityName()); + } else { + LOGGER.error("Error while creating entity {}: '{}'", def.getEntityName(), ack.getMessage()); } - public static final AttributeType mapAttributeType(AttributeDefinition.AttributeType type) { - switch (type) { - case AUTO: - return AttributeType.AUTO; - case BOOLEAN: - return AttributeType.BOOLEAN; - case DOUBLE: - return AttributeType.DOUBLE; - case VECTOR: - return AttributeType.VECTOR; - case FLOAT: - return AttributeType.FLOAT; - case GEOGRAPHY: - return AttributeType.GEOGRAPHY; - case GEOMETRY: - return AttributeType.GEOMETRY; - case INT: - return AttributeType.INT; - case LONG: - return AttributeType.LONG; - case STRING: - return AttributeType.STRING; - case TEXT: - return AttributeType.TEXT; - default: - return AttributeType.UNKOWNAT; - } + return ack.getCode() == Code.OK; + } + + /* (non-Javadoc) + * @see org.vitrivr.cineast.core.db.setup.IEntityCreator#existsEntity(java.lang.String) + */ + @Override + public boolean existsEntity(String entityName) { + return this.adampro.existsEntityBlocking(entityName); + } + + @Override + public boolean dropEntity(String entityName) { + return this.adampro.dropEntityBlocking(entityName); + } + + @Override + public boolean createHashNonUniqueIndex(String entityName, String column) { + /* If you are ever going to benchmark adampro vs cottontail, you should implement this method*/ + throw new UnsupportedOperationException(); + } + + public boolean dropIndex(String indexName) { + return this.adampro.dropIndexBlocking(indexName); + } + + /* (non-Javadoc) + * @see org.vitrivr.cineast.core.db.setup.IEntityCreator#close() + */ + @Override + public void close() { + //this.adampro.close(); + } + + public static final AttributeType mapAttributeType(AttributeDefinition.AttributeType type) { + switch (type) { + case AUTO: + return AttributeType.AUTO; + case BOOLEAN: + return AttributeType.BOOLEAN; + case DOUBLE: + return AttributeType.DOUBLE; + case VECTOR: + return AttributeType.VECTOR; + case FLOAT: + return AttributeType.FLOAT; + case GEOGRAPHY: + return AttributeType.GEOGRAPHY; + case GEOMETRY: + return AttributeType.GEOMETRY; + case INT: + return AttributeType.INT; + case LONG: + return AttributeType.LONG; + case STRING: + return AttributeType.STRING; + case TEXT: + return AttributeType.TEXT; + default: + return AttributeType.UNKOWNAT; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java index 0a80d7c39..b80527fc9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java @@ -1,412 +1,426 @@ package org.vitrivr.cineast.core.db.adampro; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.vitrivr.adampro.grpc.AdamGrpc; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.BatchedQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage; import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DistanceMessage; import org.vitrivr.adampro.grpc.AdamGrpc.DistanceMessage.DistanceType; +import org.vitrivr.adampro.grpc.AdamGrpc.ExpressionQueryMessage; import org.vitrivr.adampro.grpc.AdamGrpc.ExpressionQueryMessage.Operation; +import org.vitrivr.adampro.grpc.AdamGrpc.FromMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.NearestNeighbourQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.ProjectionMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.SubExpressionQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.VectorMessage; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - public class ADAMproMessageBuilder { - /** Default projection message.. */ - public static final ProjectionMessage DEFAULT_PROJECTION_MESSAGE; - - /** Default query hint used in absence of any explicit hint. */ - public static final Collection DEFAULT_HINT = new ArrayList(1); - - /** Supported distance messages. */ - private static final DistanceMessage chisquared, correlation, cosine, hamming, jaccard, - kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, spannorm, haversine; - - static { - DEFAULT_PROJECTION_MESSAGE = AdamGrpc.ProjectionMessage.newBuilder().setAttributes(AdamGrpc.ProjectionMessage.AttributeNameMessage.newBuilder().addAttribute("ap_distance").addAttribute("id")).build(); - DEFAULT_HINT.add(ReadableQueryConfig.Hints.exact); - - DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder(); - - chisquared = dmBuilder.clear().setDistancetype(DistanceType.chisquared).build(); - correlation = dmBuilder.clear().setDistancetype(DistanceType.correlation).build(); - cosine = dmBuilder.clear().setDistancetype(DistanceType.cosine).build(); - hamming = dmBuilder.clear().setDistancetype(DistanceType.hamming).build(); - jaccard = dmBuilder.clear().setDistancetype(DistanceType.jaccard).build(); - kullbackleibler = dmBuilder.clear().setDistancetype(DistanceType.kullbackleibler).build(); - chebyshev = dmBuilder.clear().setDistancetype(DistanceType.chebyshev).build(); - euclidean = dmBuilder.clear().setDistancetype(DistanceType.euclidean).build(); - squaredeuclidean = dmBuilder.clear().setDistancetype(DistanceType.squaredeuclidean).build(); - manhattan = dmBuilder.clear().setDistancetype(DistanceType.manhattan).build(); - spannorm = dmBuilder.clear().setDistancetype(DistanceType.spannorm).build(); - haversine = dmBuilder.clear().setDistancetype(DistanceType.haversine).build(); + /** + * Default projection message.. + */ + public static final ProjectionMessage DEFAULT_PROJECTION_MESSAGE; + + /** + * Default query hint used in absence of any explicit hint. + */ + public static final Collection DEFAULT_HINT = new ArrayList(1); + + /** + * Supported distance messages. + */ + private static final DistanceMessage chisquared, correlation, cosine, hamming, jaccard, + kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, spannorm, haversine; + + static { + DEFAULT_PROJECTION_MESSAGE = AdamGrpc.ProjectionMessage.newBuilder().setAttributes(AdamGrpc.ProjectionMessage.AttributeNameMessage.newBuilder().addAttribute("ap_distance").addAttribute("id")).build(); + DEFAULT_HINT.add(ReadableQueryConfig.Hints.exact); + + DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder(); + + chisquared = dmBuilder.clear().setDistancetype(DistanceType.chisquared).build(); + correlation = dmBuilder.clear().setDistancetype(DistanceType.correlation).build(); + cosine = dmBuilder.clear().setDistancetype(DistanceType.cosine).build(); + hamming = dmBuilder.clear().setDistancetype(DistanceType.hamming).build(); + jaccard = dmBuilder.clear().setDistancetype(DistanceType.jaccard).build(); + kullbackleibler = dmBuilder.clear().setDistancetype(DistanceType.kullbackleibler).build(); + chebyshev = dmBuilder.clear().setDistancetype(DistanceType.chebyshev).build(); + euclidean = dmBuilder.clear().setDistancetype(DistanceType.euclidean).build(); + squaredeuclidean = dmBuilder.clear().setDistancetype(DistanceType.squaredeuclidean).build(); + manhattan = dmBuilder.clear().setDistancetype(DistanceType.manhattan).build(); + spannorm = dmBuilder.clear().setDistancetype(DistanceType.spannorm).build(); + haversine = dmBuilder.clear().setDistancetype(DistanceType.haversine).build(); + } + + private final AdamGrpc.FromMessage.Builder fromBuilder = AdamGrpc.FromMessage.newBuilder(); + private final AdamGrpc.QueryMessage.Builder qmBuilder = AdamGrpc.QueryMessage.newBuilder(); + private final AdamGrpc.SubExpressionQueryMessage.Builder seqmBuilder = AdamGrpc.SubExpressionQueryMessage.newBuilder(); + private final AdamGrpc.ExpressionQueryMessage.Builder eqmBuilder = AdamGrpc.ExpressionQueryMessage.newBuilder(); + private final AdamGrpc.BatchedQueryMessage.Builder baqmBuilder = AdamGrpc.BatchedQueryMessage.newBuilder(); + private final AdamGrpc.NearestNeighbourQueryMessage.Builder nnqmBuilder = AdamGrpc.NearestNeighbourQueryMessage.newBuilder(); + private final AdamGrpc.BooleanQueryMessage.Builder bqmBuilder = AdamGrpc.BooleanQueryMessage.newBuilder(); + private final AdamGrpc.BooleanQueryMessage.WhereMessage.Builder wmBuilder = AdamGrpc.BooleanQueryMessage.WhereMessage.newBuilder(); + private final AdamGrpc.DistanceMessage.Builder dmBuilder = AdamGrpc.DistanceMessage.newBuilder(); + + /** + * Constructs and returns a BatchedQueryMessage from the provided query-messages. + */ + public BatchedQueryMessage buildBatchedQueryMessage(List queries) { + synchronized (bqmBuilder) { + baqmBuilder.clear(); + baqmBuilder.addAllQueries(queries); + return baqmBuilder.build(); } - - private final AdamGrpc.FromMessage.Builder fromBuilder = AdamGrpc.FromMessage.newBuilder(); - private final AdamGrpc.QueryMessage.Builder qmBuilder = AdamGrpc.QueryMessage.newBuilder(); - private final AdamGrpc.SubExpressionQueryMessage.Builder seqmBuilder = AdamGrpc.SubExpressionQueryMessage.newBuilder(); - private final AdamGrpc.ExpressionQueryMessage.Builder eqmBuilder = AdamGrpc.ExpressionQueryMessage.newBuilder(); - private final AdamGrpc.BatchedQueryMessage.Builder baqmBuilder = AdamGrpc.BatchedQueryMessage.newBuilder(); - private final AdamGrpc.NearestNeighbourQueryMessage.Builder nnqmBuilder = AdamGrpc.NearestNeighbourQueryMessage.newBuilder(); - private final AdamGrpc.BooleanQueryMessage.Builder bqmBuilder = AdamGrpc.BooleanQueryMessage.newBuilder(); - private final AdamGrpc.BooleanQueryMessage.WhereMessage.Builder wmBuilder = AdamGrpc.BooleanQueryMessage.WhereMessage.newBuilder(); - private final AdamGrpc.DistanceMessage.Builder dmBuilder = AdamGrpc.DistanceMessage.newBuilder(); - - /** - * Constructs and returns a BatchedQueryMessage from the provided query-messages. - */ - public BatchedQueryMessage buildBatchedQueryMessage(List queries) { - synchronized (bqmBuilder) { - baqmBuilder.clear(); - baqmBuilder.addAllQueries(queries); - return baqmBuilder.build(); - } + } + + + public QueryMessage buildQueryMessage(Collection hints, FromMessage.Builder fb, + BooleanQueryMessage bqMessage, ProjectionMessage pMessage, + NearestNeighbourQueryMessage nnqMessage) { + synchronized (qmBuilder) { + qmBuilder.clear(); + qmBuilder.setFrom(fb); + if (hints != null && !hints.isEmpty()) { + qmBuilder.addAllHints(hints.stream().map(Enum::name).collect(Collectors.toList())); + } + if (bqMessage != null) { + qmBuilder.setBq(bqMessage); + } + if (pMessage != null) { + qmBuilder.setProjection(pMessage); + } + if (nnqMessage != null) { + qmBuilder.setNnq(nnqMessage); + } + return qmBuilder.build(); } - - - - public QueryMessage buildQueryMessage(Collection hints, FromMessage.Builder fb, - BooleanQueryMessage bqMessage, ProjectionMessage pMessage, - NearestNeighbourQueryMessage nnqMessage) { - synchronized (qmBuilder) { - qmBuilder.clear(); - qmBuilder.setFrom(fb); - if (hints != null && !hints.isEmpty()) { - qmBuilder.addAllHints(hints.stream().map(Enum::name).collect(Collectors.toList())); - } - if (bqMessage != null) { - qmBuilder.setBq(bqMessage); - } - if (pMessage != null) { - qmBuilder.setProjection(pMessage); - } - if (nnqMessage != null) { - qmBuilder.setNnq(nnqMessage); - } - return qmBuilder.build(); - } + } + + public QueryMessage buildQueryMessage(Collection hints, FromMessage fromMessage, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage) { + synchronized (this.qmBuilder) { + this.qmBuilder.clear(); + this.qmBuilder.setFrom(fromMessage); + if (hints != null && !hints.isEmpty()) { + qmBuilder.addAllHints(hints.stream().map(Enum::name).collect(Collectors.toList())); + } + if (bqMessage != null) { + this.qmBuilder.setBq(bqMessage); + } + if (pMessage != null) { + this.qmBuilder.setProjection(pMessage); + } + if (nnqMessage != null) { + this.qmBuilder.setNnq(nnqMessage); + } + return qmBuilder.build(); } + } - public QueryMessage buildQueryMessage(Collection hints, FromMessage fromMessage, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage) { - synchronized (this.qmBuilder) { - this.qmBuilder.clear(); - this.qmBuilder.setFrom(fromMessage); - if (hints != null && !hints.isEmpty()) { - qmBuilder.addAllHints(hints.stream().map(Enum::name).collect(Collectors.toList())); - } - if (bqMessage != null) { - this.qmBuilder.setBq(bqMessage); - } - if (pMessage != null) { - this.qmBuilder.setProjection(pMessage); - } - if (nnqMessage != null) { - this.qmBuilder.setNnq(nnqMessage); - } - return qmBuilder.build(); - } + public FromMessage buildFromMessage(String entity) { + synchronized (this.fromBuilder) { + this.fromBuilder.clear(); + this.fromBuilder.setEntity(entity); + return this.fromBuilder.build(); } - - public FromMessage buildFromMessage(String entity) { - synchronized (this.fromBuilder) { - this.fromBuilder.clear(); - this.fromBuilder.setEntity(entity); - return this.fromBuilder.build(); - } + } + + /** + * Builds a FromMessage from a SubExpressionQueryMessage. This method can be used to select from a set that has been built up previously + * + * @param message SubExpressionQueryMessage message + * @return FromMessage + */ + public FromMessage buildFromSubExpressionMessage(SubExpressionQueryMessage message) { + synchronized (this.fromBuilder) { + this.fromBuilder.clear(); + this.fromBuilder.setExpression(message); + return this.fromBuilder.build(); } - - /** - * Builds a FromMessage from a SubExpressionQueryMessage. This method can be used to select - * from a set that has been built up previously - * - * @param message SubExpressionQueryMessage message - * @return FromMessage - */ - public FromMessage buildFromSubExpressionMessage(SubExpressionQueryMessage message) { - synchronized (this.fromBuilder) { - this.fromBuilder.clear(); - this.fromBuilder.setExpression(message); - return this.fromBuilder.build(); - } + } + + /** + * Builds a SubExpressionQueryMessage from a QueryMessage. + */ + public SubExpressionQueryMessage buildSubExpressionQueryMessage(QueryMessage message) { + synchronized (this.seqmBuilder) { + this.seqmBuilder.clear(); + this.seqmBuilder.setQm(message); + return this.seqmBuilder.build(); } - - /** - * Builds a SubExpressionQueryMessage from a QueryMessage. - */ - public SubExpressionQueryMessage buildSubExpressionQueryMessage(QueryMessage message) { - synchronized (this.seqmBuilder) { - this.seqmBuilder.clear(); - this.seqmBuilder.setQm(message); - return this.seqmBuilder.build(); - } + } + + /** + * Builds a SubExpressionQueryMessage from a ExpressionQueryMessage. + */ + public SubExpressionQueryMessage buildSubExpressionQueryMessage(ExpressionQueryMessage message) { + synchronized (this.seqmBuilder) { + this.seqmBuilder.clear(); + this.seqmBuilder.setEqm(message); + return this.seqmBuilder.build(); } - - /** - * Builds a SubExpressionQueryMessage from a ExpressionQueryMessage. - */ - public SubExpressionQueryMessage buildSubExpressionQueryMessage(ExpressionQueryMessage message) { - synchronized (this.seqmBuilder) { - this.seqmBuilder.clear(); - this.seqmBuilder.setEqm(message); - return this.seqmBuilder.build(); - } + } + + /** + * Builds an ExpressionQueryMesssage, that is a QueryMessage that combines the results of two SubExpressionQuerymessages using a Set operation. + * + * @param left First SubExpressionQueryMessage to combine. + * @param right Second SubExpressionQueryMessage to combine. + * @param operation Set operation. + * @param options Named options that should be passed to the ExpressionQueryMessage. + * @return ExpressionQueryMessage + */ + public AdamGrpc.ExpressionQueryMessage buildExpressionQueryMessage(SubExpressionQueryMessage left, SubExpressionQueryMessage right, Operation operation, Map options) { + synchronized (this.eqmBuilder) { + this.eqmBuilder.clear(); + this.eqmBuilder.setLeft(left); + this.eqmBuilder.setRight(right); + this.eqmBuilder.setOperation(operation); + this.eqmBuilder.setOrder(ExpressionQueryMessage.OperationOrder.PARALLEL); + if (options != null && options.size() > 0) { + this.eqmBuilder.putAllOptions(options); + } + return this.eqmBuilder.build(); } - - /** - * Builds an ExpressionQueryMesssage, that is a QueryMessage that combines the results of two - * SubExpressionQuerymessages using a Set operation. - * - * @param left First SubExpressionQueryMessage to combine. - * @param right Second SubExpressionQueryMessage to combine. - * @param operation Set operation. - * @param options Named options that should be passed to the ExpressionQueryMessage. - * @return ExpressionQueryMessage - */ - public AdamGrpc.ExpressionQueryMessage buildExpressionQueryMessage(SubExpressionQueryMessage left, SubExpressionQueryMessage right, Operation operation, Map options) { - synchronized (this.eqmBuilder) { - this.eqmBuilder.clear(); - this.eqmBuilder.setLeft(left); - this.eqmBuilder.setRight(right); - this.eqmBuilder.setOperation(operation); - this.eqmBuilder.setOrder(ExpressionQueryMessage.OperationOrder.PARALLEL); - if (options != null && options.size() > 0) { - this.eqmBuilder.putAllOptions(options); - } - return this.eqmBuilder.build(); - } + } + + /** + * This method recursively combines a list of SubExpressionQueryMessages into a single SubExpressionQueryMessage by building corresponding ExpressionQueryMessage. Calling this method for a list of SubExpressionQueryMessage creates a new SubExpressionQueryMessage that combines the query results of each SubExpressionQueryMessage under the provided operation. + * + * @param expressions List of SubExpressionQueryMessages + * @param operation Set operation used for combining partial results + */ + public SubExpressionQueryMessage mergeSubexpressions(List expressions, Operation operation, Map options) { + /* If list only contains one SubExpressionQueryMessage then return it. */ + if (expressions.size() == 1) { + return expressions.get(0); } - /** - * This method recursively combines a list of SubExpressionQueryMessages into a single SubExpressionQueryMessage by - * building corresponding ExpressionQueryMessage. Calling this method for a list of SubExpressionQueryMessage creates - * a new SubExpressionQueryMessage that combines the query results of each SubExpressionQueryMessage under the - * provided operation. - * - * @param expressions List of SubExpressionQueryMessages - * @param operation Set operation used for combining partial results - */ - public SubExpressionQueryMessage mergeSubexpressions(List expressions, Operation operation, Map options) { - /* If list only contains one SubExpressionQueryMessage then return it. */ - if (expressions.size() == 1) { - return expressions.get(0); - } - - /* Take first and second message and remove them from the list. */ - SubExpressionQueryMessage m1 = expressions.get(0); - SubExpressionQueryMessage m2 = expressions.get(1); - expressions.remove(0); - expressions.remove(0); - - /* Merge expressions into an ExpressionQueryMessage using the operation and add them to the list. */ - ExpressionQueryMessage eqm = buildExpressionQueryMessage(m1, m2, operation, options); - SubExpressionQueryMessage sqm = buildSubExpressionQueryMessage(eqm); - expressions.add(sqm); - - /* Call again. */ - return mergeSubexpressions(expressions, operation, options); + /* Take first and second message and remove them from the list. */ + SubExpressionQueryMessage m1 = expressions.get(0); + SubExpressionQueryMessage m2 = expressions.get(1); + expressions.remove(0); + expressions.remove(0); + + /* Merge expressions into an ExpressionQueryMessage using the operation and add them to the list. */ + ExpressionQueryMessage eqm = buildExpressionQueryMessage(m1, m2, operation, options); + SubExpressionQueryMessage sqm = buildSubExpressionQueryMessage(eqm); + expressions.add(sqm); + + /* Call again. */ + return mergeSubexpressions(expressions, operation, options); + } + + public BooleanQueryMessage buildBooleanQueryMessage(WhereMessage where, + WhereMessage... whereMessages) { + ArrayList tmp = new ArrayList<>( + 1 + (whereMessages == null ? 0 : whereMessages.length)); + tmp.add(where); + if (whereMessages != null) { + Collections.addAll(tmp, whereMessages); } - - public BooleanQueryMessage buildBooleanQueryMessage(WhereMessage where, - WhereMessage... whereMessages) { - ArrayList tmp = new ArrayList<>( - 1 + (whereMessages == null ? 0 : whereMessages.length)); - tmp.add(where); - if (whereMessages != null) { - Collections.addAll(tmp, whereMessages); - } - synchronized (bqmBuilder) { - bqmBuilder.clear(); - return bqmBuilder.addAllWhere(tmp).build(); - } + synchronized (bqmBuilder) { + bqmBuilder.clear(); + return bqmBuilder.addAllWhere(tmp).build(); } - - /** - * Builds a {@link WhereMessage} using the specified settings and the equals operator. - * - * @param key The name of the field (key). - * @param value The values that should be compared against the field. - * @return {@link WhereMessage} - */ - public WhereMessage buildWhereMessage(String key, String value) { - return buildWhereMessage(key, Collections.singleton(value)); + } + + /** + * Builds a {@link WhereMessage} using the specified settings and the equals operator. + * + * @param key The name of the field (key). + * @param value The values that should be compared against the field. + * @return {@link WhereMessage} + */ + public WhereMessage buildWhereMessage(String key, String value) { + return buildWhereMessage(key, Collections.singleton(value)); + } + + /** + * Builds a {@link WhereMessage} using the specified settings and the equals operator. + * + * @param key The name of the field (key). + * @param values The list of values that should be compared against the field. + * @return {@link WhereMessage} + */ + public WhereMessage buildWhereMessage(String key, Iterable values) { + return this.buildWhereMessage(key, values, RelationalOperator.EQ); + } + + /** + * Builds a {@link WhereMessage} using the specified settings. + * + * @param key The name of the field (key). + * @param values The list of values that should be compared against the field. + * @param operator The {@link RelationalOperator} used to compare the field (key) and the values. + * @return {@link WhereMessage} + */ + public WhereMessage buildWhereMessage(String key, Iterable values, RelationalOperator operator) { + synchronized (this.wmBuilder) { + this.wmBuilder.clear(); + final DataMessage.Builder damBuilder = DataMessage.newBuilder(); + final Stream valueStream = StreamSupport.stream(values.spliterator(), false); + switch (operator) { + case IN: + case EQ: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("="); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case NEQ: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("!="); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case GEQ: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp(">="); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case LEQ: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("<="); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case GREATER: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("<"); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case LESS: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp(">"); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); + break; + case LIKE: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("LIKE"); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData("%" + v + "%").build()).collect(Collectors.toList())); + break; + case NLIKE: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("NOT LIKE"); + this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData("%" + v + "%").build()).collect(Collectors.toList())); + break; + case MATCH: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("MATCH"); + break; + default: + this.wmBuilder.setAttribute(key); + this.wmBuilder.setOp("="); + break; + } + return this.wmBuilder.build(); } - - /** - * Builds a {@link WhereMessage} using the specified settings and the equals operator. - * - * @param key The name of the field (key). - * @param values The list of values that should be compared against the field. - * @return {@link WhereMessage} - */ - public WhereMessage buildWhereMessage(String key, Iterable values) { - return this.buildWhereMessage(key, values, RelationalOperator.EQ); + } + + public NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String column, VectorMessage fvm, int k, ReadableQueryConfig qc) { + synchronized (nnqmBuilder) { + this.nnqmBuilder.clear(); + this.nnqmBuilder.setAttribute(column).setQuery(fvm).setK(k); + this.nnqmBuilder.setDistance(buildDistanceMessage(qc)); + if (qc != null) { + Optional weights = qc.getDistanceWeights(); + weights.ifPresent(floats -> nnqmBuilder + .setWeights(DataMessageConverter.convertVectorMessage(floats))); + } + return nnqmBuilder.build(); } + } - /** - * Builds a {@link WhereMessage} using the specified settings. - * - * @param key The name of the field (key). - * @param values The list of values that should be compared against the field. - * @param operator The {@link RelationalOperator} used to compare the field (key) and the values. - * @return {@link WhereMessage} - */ - public WhereMessage buildWhereMessage(String key, Iterable values, RelationalOperator operator) { - synchronized (this.wmBuilder) { - this.wmBuilder.clear(); - final DataMessage.Builder damBuilder = DataMessage.newBuilder(); - final Stream valueStream = StreamSupport.stream(values.spliterator(), false); - switch (operator) { - case IN: - case EQ: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("="); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case NEQ: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("!="); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case GEQ: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp(">="); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case LEQ: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("<="); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case GREATER: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("<"); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case LESS: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp(">"); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData(v).build()).collect(Collectors.toList())); - break; - case LIKE: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("LIKE"); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData("%" + v + "%").build()).collect(Collectors.toList())); - break; - case NLIKE: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("NOT LIKE"); - this.wmBuilder.addAllValues(valueStream.map(v -> damBuilder.setStringData("%" + v + "%").build()).collect(Collectors.toList())); - break; - case MATCH: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("MATCH"); - break; - default: - this.wmBuilder.setAttribute(key); - this.wmBuilder.setOp("="); - break; - } - return this.wmBuilder.build(); - } + public DistanceMessage buildDistanceMessage(ReadableQueryConfig qc) { + if (qc == null) { + return manhattan; } - - public NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String column, VectorMessage fvm, int k, ReadableQueryConfig qc) { - synchronized (nnqmBuilder) { - this.nnqmBuilder.clear(); - this.nnqmBuilder.setAttribute(column).setQuery(fvm).setK(k); - this.nnqmBuilder.setDistance(buildDistanceMessage(qc)); - if (qc != null) { - Optional weights = qc.getDistanceWeights(); - weights.ifPresent(floats -> nnqmBuilder - .setWeights(DataMessageConverter.convertVectorMessage(floats))); - } - return nnqmBuilder.build(); - } + Optional distance = qc.getDistance(); + if (!distance.isPresent()) { + return manhattan; } - - public DistanceMessage buildDistanceMessage(ReadableQueryConfig qc) { - if (qc == null) { - return manhattan; - } - Optional distance = qc.getDistance(); - if (!distance.isPresent()) { - return manhattan; + switch (distance.get()) { + case chebyshev: + return chebyshev; + case chisquared: + return chisquared; + case correlation: + return correlation; + case cosine: + return cosine; + case euclidean: + return euclidean; + case hamming: + return hamming; + case jaccard: + return jaccard; + case kullbackleibler: + return kullbackleibler; + case manhattan: + return manhattan; + case minkowski: { + + float norm = qc.getNorm().orElse(1f); + + if (Math.abs(norm - 1f) < 1e-6f) { + return manhattan; } - switch (distance.get()) { - case chebyshev: - return chebyshev; - case chisquared: - return chisquared; - case correlation: - return correlation; - case cosine: - return cosine; - case euclidean: - return euclidean; - case hamming: - return hamming; - case jaccard: - return jaccard; - case kullbackleibler: - return kullbackleibler; - case manhattan: - return manhattan; - case minkowski: { - - float norm = qc.getNorm().orElse(1f); - - if (Math.abs(norm - 1f) < 1e-6f) { - return manhattan; - } - - if (Math.abs(norm - 2f) < 1e-6f) { - return euclidean; - } - - HashMap tmp = new HashMap<>(); - tmp.put("norm", Float.toString(norm)); - - synchronized (dmBuilder) { - return dmBuilder.clear().setDistancetype(DistanceType.minkowski).putAllOptions(tmp) - .build(); - } - - } - case spannorm: - return spannorm; - case squaredeuclidean: - return squaredeuclidean; - case haversine: - return haversine; - default: - return manhattan; + + if (Math.abs(norm - 2f) < 1e-6f) { + return euclidean; } - } - public BooleanQueryMessage inList(String attribute, Collection elements) { + HashMap tmp = new HashMap<>(); + tmp.put("norm", Float.toString(norm)); - if (elements == null || elements.isEmpty()) { - return null; + synchronized (dmBuilder) { + return dmBuilder.clear().setDistancetype(DistanceType.minkowski).putAllOptions(tmp) + .build(); } - final DataMessage.Builder damBuilder = DataMessage.newBuilder(); + } + case spannorm: + return spannorm; + case squaredeuclidean: + return squaredeuclidean; + case haversine: + return haversine; + default: + return manhattan; + } + } - return BooleanQueryMessage.newBuilder() - .addWhere( - WhereMessage.newBuilder() - .setOp("IN") - .setAttribute(attribute) - .addAllValues( - elements.stream().map(x -> damBuilder.clear().setStringData(x).build()).collect(Collectors.toList()) - ) - .build() - ).build(); + public BooleanQueryMessage inList(String attribute, Collection elements) { + if (elements == null || elements.isEmpty()) { + return null; } + + final DataMessage.Builder damBuilder = DataMessage.newBuilder(); + + return BooleanQueryMessage.newBuilder() + .addWhere( + WhereMessage.newBuilder() + .setOp("IN") + .setAttribute(attribute) + .addAllValues( + elements.stream().map(x -> damBuilder.clear().setStringData(x).build()).collect(Collectors.toList()) + ) + .build() + ).build(); + + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java index 64e9aad24..7fbbe7512 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java @@ -2,13 +2,33 @@ import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage.Code; +import org.vitrivr.adampro.grpc.AdamGrpc.BatchedQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.BatchedQueryResultsMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage; import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DenseVectorMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.ExternalHandlerQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.FromMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.NearestNeighbourQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultInfoMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultTupleMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultsMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.SubExpressionQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.VectorMessage; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.DistanceElement; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; @@ -16,325 +36,319 @@ import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.util.LogHelper; -import java.util.*; -import java.util.concurrent.ExecutionException; - public class ADAMproSelector extends AbstractADAMproSelector { - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); + + public ADAMproSelector(ADAMproWrapper wrapper) { + super(wrapper); + } - public ADAMproSelector(ADAMproWrapper wrapper) { - super(wrapper); + @Override + public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { + QueryMessage qbqm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, this.fromMessage, this.mb.buildBooleanQueryMessage(this.mb.buildWhereMessage(fieldName, value.getString())), null, null); + + ListenableFuture f = this.adampro.booleanQuery(qbqm); + ArrayList _return = new ArrayList<>(); + QueryResultsMessage r; + try { + r = f.get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + return new ArrayList<>(0); } - @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { - QueryMessage qbqm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, this.fromMessage, this.mb.buildBooleanQueryMessage(this.mb.buildWhereMessage(fieldName, value.getString())), null, null); + if (r.getResponsesCount() == 0) { + return new ArrayList<>(0); + } - ListenableFuture f = this.adampro.booleanQuery(qbqm); - ArrayList _return = new ArrayList<>(); - QueryResultsMessage r; - try { - r = f.get(); - } catch (InterruptedException | ExecutionException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - return new ArrayList<>(0); - } + QueryResultInfoMessage response = r.getResponses(0); // only head (end-result) is important - if (r.getResponsesCount() == 0) { - return new ArrayList<>(0); - } + AckMessage ack = response.getAck(); + if (ack.getCode() != Code.OK) { + LOGGER.error("error in getFeatureVectors on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); + return _return; + } - QueryResultInfoMessage response = r.getResponses(0); // only head (end-result) is important + for (QueryResultTupleMessage result : response.getResultsList()) { - AckMessage ack = response.getAck(); - if (ack.getCode() != Code.OK) { - LOGGER.error("error in getFeatureVectors on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - return _return; - } + Map data = result.getDataMap(); - for (QueryResultTupleMessage result : response.getResultsList()) { + if (!data.containsKey(vectorName)) { + continue; + } - Map data = result.getDataMap(); + DataMessage dm = data.get(vectorName); - if (!data.containsKey(vectorName)) { - continue; - } + if (dm.getDatatypeCase() != DataMessage.DatatypeCase.VECTORDATA) { + continue; + } - DataMessage dm = data.get(vectorName); + VectorMessage featureData = dm.getVectorData(); - if (dm.getDatatypeCase() != DataMessage.DatatypeCase.VECTORDATA) { - continue; - } + if (featureData.getVectorCase() != VectorMessage.VectorCase.DENSEVECTOR) { + continue; // TODO add correct handling for sparse and int vectors + } - VectorMessage featureData = dm.getVectorData(); + DenseVectorMessage dense = featureData.getDenseVector(); - if (featureData.getVectorCase() != VectorMessage.VectorCase.DENSEVECTOR) { - continue; // TODO add correct handling for sparse and int vectors - } + List list = dense.getVectorList(); + if (list.isEmpty()) { + continue; + } - DenseVectorMessage dense = featureData.getDenseVector(); + float[] vector = new float[list.size()]; + int i = 0; + for (float x : list) { + vector[i++] = x; + } - List list = dense.getVectorList(); - if (list.isEmpty()) { - continue; - } + _return.add(vector); - float[] vector = new float[list.size()]; - int i = 0; - for (float x : list) { - vector[i++] = x; - } + } + + return _return; + + } + + /** + * Performs a batched kNN-search with multiple vectors. That is, ADAM pro is tasked to perform the kNN search for each vector in the provided list and return results of each query. + * + * @param k The number k vectors to return per query. + * @param vectors The list of vectors to use. + * @param column The column to perform the kNN search on. + * @param distanceElementClass The class to use to create the resulting DistanceElements + * @param configs The query configurations, which may contain distance definitions or query-hints. Every feature should have its own QueryConfig object. + * @param The type T of the resulting DistanceElements. + * @return List of results. + */ + @Override + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + /* Check if sizes of configs and vectors array correspond. */ + if (vectors.size() > configs.size()) { + throw new IllegalArgumentException("You must provide a separate QueryConfig entry for each vector - even if it is the same instance of the QueryConfig."); + } - _return.add(vector); + /* Prepare list of QueryMessages. */ + List queryMessages = new ArrayList<>(vectors.size()); + for (int i = 0; i < vectors.size(); i++) { + float[] vector = vectors.get(i); + ReadableQueryConfig config = configs.get(i); + + /* Extract hints from QueryConfig. If they're not set, then replace by DEFAULT_HINT. */ + Collection hints; + if (!config.getHints().isEmpty()) { + hints = config.getHints(); + } else { + hints = ADAMproMessageBuilder.DEFAULT_HINT; + } + + NearestNeighbourQueryMessage nnqMessage = this.mb.buildNearestNeighbourQueryMessage(column, DataMessageConverter.convertVectorMessage(vector), k, config); + queryMessages.add(this.mb.buildQueryMessage(hints, this.fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage)); + } - } + /* Prepare a BatchedQueryMessage. */ + BatchedQueryMessage batchedQueryMessage = this.mb.buildBatchedQueryMessage(queryMessages); - return _return; + ListenableFuture future = this.adampro.batchedQuery(batchedQueryMessage); + BatchedQueryResultsMessage result; + try { + result = future.get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + return new ArrayList<>(0); } - /** - * Performs a batched kNN-search with multiple vectors. That is, ADAM pro is tasked to perform the kNN search for each vector in the - * provided list and return results of each query. - * - * @param k The number k vectors to return per query. - * @param vectors The list of vectors to use. - * @param column The column to perform the kNN search on. - * @param distanceElementClass The class to use to create the resulting DistanceElements - * @param configs The query configurations, which may contain distance definitions or query-hints. Every feature should have its own QueryConfig object. - * @param The type T of the resulting DistanceElements. - * @return List of results. + /* Prepare empty list of results. */ + List results = new ArrayList<>(result.getResultsCount()); + + /* + * Merge results of the partial queries. */ - @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { - /* Check if sizes of configs and vectors array correspond. */ - if (vectors.size() > configs.size()) { - throw new IllegalArgumentException("You must provide a separate QueryConfig entry for each vector - even if it is the same instance of the QueryConfig."); - } - - /* Prepare list of QueryMessages. */ - List queryMessages = new ArrayList<>(vectors.size()); - for (int i = 0; i < vectors.size(); i++) { - float[] vector = vectors.get(i); - ReadableQueryConfig config = configs.get(i); - - /* Extract hints from QueryConfig. If they're not set, then replace by DEFAULT_HINT. */ - Collection hints; - if (!config.getHints().isEmpty()) { - hints = config.getHints(); - } else { - hints = ADAMproMessageBuilder.DEFAULT_HINT; - } - - NearestNeighbourQueryMessage nnqMessage = this.mb.buildNearestNeighbourQueryMessage(column, DataMessageConverter.convertVectorMessage(vector), k, config); - queryMessages.add(this.mb.buildQueryMessage(hints, this.fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage)); - } - - /* Prepare a BatchedQueryMessage. */ - BatchedQueryMessage batchedQueryMessage = this.mb.buildBatchedQueryMessage(queryMessages); - - ListenableFuture future = this.adampro.batchedQuery(batchedQueryMessage); - - BatchedQueryResultsMessage result; - try { - result = future.get(); - } catch (InterruptedException | ExecutionException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - return new ArrayList<>(0); - } - - /* Prepare empty list of results. */ - List results = new ArrayList<>(result.getResultsCount()); - - /* - * Merge results of the partial queries. - */ - for (int i = 0; i < result.getResultsCount(); i++) { - QueryResultsMessage partial = result.getResults(i); - AckMessage ack = partial.getAck(); - if (ack.getCode() != AckMessage.Code.OK) { - LOGGER.error("error in getNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - continue; - } - - if (partial.getResponsesCount() == 0) { - continue; - } - - QueryResultInfoMessage response = partial.getResponses(0); // only head (end-result) is important - results.addAll(handleNearestNeighbourResponse(response, k, distanceElementClass)); - } - - return results; + for (int i = 0; i < result.getResultsCount(); i++) { + QueryResultsMessage partial = result.getResults(i); + AckMessage ack = partial.getAck(); + if (ack.getCode() != AckMessage.Code.OK) { + LOGGER.error("error in getNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); + continue; + } + + if (partial.getResponsesCount() == 0) { + continue; + } + + QueryResultInfoMessage response = partial.getResponses(0); // only head (end-result) is important + results.addAll(handleNearestNeighbourResponse(response, k, distanceElementClass)); } - @Override - public List getNearestNeighboursGeneric(int k, float[] vector, String column, - Class distanceElementClass, ReadableQueryConfig config) { - NearestNeighbourQueryMessage nnqMessage = mb.buildNearestNeighbourQueryMessage(column, - DataMessageConverter.convertVectorMessage(vector), k, config); - QueryMessage sqMessage = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage); - ListenableFuture future = this.adampro.standardQuery(sqMessage); - - QueryResultsMessage result; - try { - result = future.get(); - } catch (InterruptedException | ExecutionException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - return new ArrayList<>(0); - } - - AckMessage ack = result.getAck(); - if (ack.getCode() != AckMessage.Code.OK) { - LOGGER.error("error in getNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - LOGGER.error("Query was {} ",sqMessage.toString()); - return new ArrayList<>(0); - } - - if (result.getResponsesCount() == 0) { - return new ArrayList<>(0); - } - - QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important - return handleNearestNeighbourResponse(response, k, distanceElementClass); + return results; + } + + @Override + public List getNearestNeighboursGeneric(int k, float[] vector, String column, + Class distanceElementClass, ReadableQueryConfig config) { + NearestNeighbourQueryMessage nnqMessage = mb.buildNearestNeighbourQueryMessage(column, + DataMessageConverter.convertVectorMessage(vector), k, config); + QueryMessage sqMessage = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage); + ListenableFuture future = this.adampro.standardQuery(sqMessage); + + QueryResultsMessage result; + try { + result = future.get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + return new ArrayList<>(0); } + AckMessage ack = result.getAck(); + if (ack.getCode() != AckMessage.Code.OK) { + LOGGER.error("error in getNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); + LOGGER.error("Query was {} ", sqMessage.toString()); + return new ArrayList<>(0); + } - @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { - if (values == null || Iterables.isEmpty(values)) { - return new ArrayList<>(0); - } - - /* TODO: Escape quotes. */ - final WhereMessage where = this.mb.buildWhereMessage(fieldName, StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::getString).collect(Collectors.toList()), operator); - final BooleanQueryMessage bqMessage = this.mb.buildBooleanQueryMessage(where); - return executeBooleanQuery(bqMessage); + if (result.getResponsesCount() == 0) { + return new ArrayList<>(0); } + QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important + return handleNearestNeighbourResponse(response, k, distanceElementClass); + } - /** - * Executes a QueryMessage and returns the resulting tuples - * - * @return an empty ArrayList if an error happens. Else just the list of rows - */ - private List> executeQuery(QueryMessage qm) { - ListenableFuture f = this.adampro.standardQuery(qm); - QueryResultsMessage result; - try { - result = f.get(); - } catch (InterruptedException | ExecutionException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - return new ArrayList<>(0); - } - - if (result.getAck().getCode() != AckMessage.Code.OK) { - LOGGER.error("Query returned non-OK result code {} with message: {}", - result.getAck().getCode(), - result.getAck().getMessage()); - } - - if (result.getResponsesCount() == 0) { - return new ArrayList<>(0); - } - - QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important - - List resultList = response.getResultsList(); - return resultsToMap(resultList); + + @Override + public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + if (values == null || Iterables.isEmpty(values)) { + return new ArrayList<>(0); } - private List> executeBooleanQuery(BooleanQueryMessage bqm) { - QueryMessage qbqm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, this.fromMessage, bqm, null, null); - return executeQuery(qbqm); + /* TODO: Escape quotes. */ + final WhereMessage where = this.mb.buildWhereMessage(fieldName, StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::getString).collect(Collectors.toList()), operator); + final BooleanQueryMessage bqMessage = this.mb.buildBooleanQueryMessage(where); + return executeBooleanQuery(bqMessage); + } + + + /** + * Executes a QueryMessage and returns the resulting tuples + * + * @return an empty ArrayList if an error happens. Else just the list of rows + */ + private List> executeQuery(QueryMessage qm) { + ListenableFuture f = this.adampro.standardQuery(qm); + QueryResultsMessage result; + try { + result = f.get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + return new ArrayList<>(0); } + if (result.getAck().getCode() != AckMessage.Code.OK) { + LOGGER.error("Query returned non-OK result code {} with message: {}", + result.getAck().getCode(), + result.getAck().getMessage()); + } + + if (result.getResponsesCount() == 0) { + return new ArrayList<>(0); + } - @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - NearestNeighbourQueryMessage nnqMessage = this.mb.buildNearestNeighbourQueryMessage(column, - DataMessageConverter.convertVectorMessage(vector), k, config); + QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important - /* Extract hints from QueryConfig. If they're not set, then replace by DEFAULT_HINT. */ - Collection hints; - if (!config.getHints().isEmpty()) { - hints = config.getHints(); - } else { - hints = ADAMproMessageBuilder.DEFAULT_HINT; - } + List resultList = response.getResultsList(); + return resultsToMap(resultList); + } - QueryMessage sqMessage = this.mb.buildQueryMessage(hints, this.fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), null, nnqMessage); + private List> executeBooleanQuery(BooleanQueryMessage bqm) { + QueryMessage qbqm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, this.fromMessage, bqm, null, null); + return executeQuery(qbqm); + } - ListenableFuture future = this.adampro.standardQuery(sqMessage); - QueryResultsMessage result; - try { - result = future.get(); - } catch (InterruptedException | ExecutionException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - return new ArrayList<>(0); - } + @Override + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { + NearestNeighbourQueryMessage nnqMessage = this.mb.buildNearestNeighbourQueryMessage(column, + DataMessageConverter.convertVectorMessage(vector), k, config); - if (result.getResponsesCount() == 0) { - return new ArrayList<>(0); - } + /* Extract hints from QueryConfig. If they're not set, then replace by DEFAULT_HINT. */ + Collection hints; + if (!config.getHints().isEmpty()) { + hints = config.getHints(); + } else { + hints = ADAMproMessageBuilder.DEFAULT_HINT; + } - QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important + QueryMessage sqMessage = this.mb.buildQueryMessage(hints, this.fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), null, nnqMessage); - ArrayList> _return = new ArrayList<>(k); + ListenableFuture future = this.adampro.standardQuery(sqMessage); - AckMessage ack = response.getAck(); - if (ack.getCode() != Code.OK) { - LOGGER.error("error in getNearestNeighbourRows, entity {} ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - return _return; - } - return resultsToMap(response.getResultsList()); + QueryResultsMessage result; + try { + result = future.get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + return new ArrayList<>(0); } - /** - * Performs a fulltext search with multiple query terms. The underlying entity is expected to use Apache Solr as storage - * handler. If it doesn't, this method will fail! - * - * The method performs an Apache Solr lookup equivalent to: [field]: ([term1] [term2] ... [termN]). Full Lucene query syntax is supported. - * - * TODO: This is a quick & dirty solution. Should be re-engineered to fit different use-cases. - * - * @param rows The number of rows that should be returned. - * @param fieldname The field that should be used for lookup. - * @param terms The query terms. Individual terms will be connected by a logical OR. - * @return List of rows that math the fulltext search. - */ - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { - final ExternalHandlerQueryMessage.Builder ehqmBuilder = ExternalHandlerQueryMessage.newBuilder(); - ehqmBuilder.setEntity(this.entityName); - ehqmBuilder.setHandler("solr"); + if (result.getResponsesCount() == 0) { + return new ArrayList<>(0); + } - /* */ - final Map parameters = new HashMap<>(); - parameters.put("rows", Integer.toString(rows)); + QueryResultInfoMessage response = result.getResponses(0); // only head (end-result) is important + ArrayList> _return = new ArrayList<>(k); - final StringBuilder sb = new StringBuilder(); - sb.append('('); - for (String item : terms) { - sb.append(item); - sb.append(" "); - } - sb.append(')'); - parameters.put("query", fieldname + ":" + sb.toString()); + AckMessage ack = response.getAck(); + if (ack.getCode() != Code.OK) { + LOGGER.error("error in getNearestNeighbourRows, entity {} ({}) : {}", entityName, ack.getCode(), ack.getMessage()); + return _return; + } + return resultsToMap(response.getResultsList()); + } + + /** + * Performs a fulltext search with multiple query terms. The underlying entity is expected to use Apache Solr as storage handler. If it doesn't, this method will fail! + *

    + * The method performs an Apache Solr lookup equivalent to: [field]: ([term1] [term2] ... [termN]). Full Lucene query syntax is supported. + *

    + * TODO: This is a quick & dirty solution. Should be re-engineered to fit different use-cases. + * + * @param rows The number of rows that should be returned. + * @param fieldname The field that should be used for lookup. + * @param terms The query terms. Individual terms will be connected by a logical OR. + * @return List of rows that math the fulltext search. + */ + public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + final ExternalHandlerQueryMessage.Builder ehqmBuilder = ExternalHandlerQueryMessage.newBuilder(); + ehqmBuilder.setEntity(this.entityName); + ehqmBuilder.setHandler("solr"); + + /* */ + final Map parameters = new HashMap<>(); + parameters.put("rows", Integer.toString(rows)); + + final StringBuilder sb = new StringBuilder(); + sb.append('('); + for (String item : terms) { + sb.append(item); + sb.append(" "); + } + sb.append(')'); + parameters.put("query", fieldname + ":" + sb.toString()); - ehqmBuilder.putAllParams(parameters); + ehqmBuilder.putAllParams(parameters); - SubExpressionQueryMessage.Builder seqmBuilder = SubExpressionQueryMessage.newBuilder(); - seqmBuilder.setEhqm(ehqmBuilder); + SubExpressionQueryMessage.Builder seqmBuilder = SubExpressionQueryMessage.newBuilder(); + seqmBuilder.setEhqm(ehqmBuilder); - FromMessage.Builder fmBuilder = FromMessage.newBuilder(); - fmBuilder.setExpression(seqmBuilder); + FromMessage.Builder fmBuilder = FromMessage.newBuilder(); + fmBuilder.setExpression(seqmBuilder); - QueryMessage qm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fmBuilder, this.mb.inList("id", queryConfig.getRelevantSegmentIds()), null, null); + QueryMessage qm = this.mb.buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fmBuilder, this.mb.inList("id", queryConfig.getRelevantSegmentIds()), null, null); - return executeQuery(qm); - } + return executeQuery(qm); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java index db52c917c..a61ee133f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java @@ -1,21 +1,37 @@ package org.vitrivr.cineast.core.db.adampro; import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage.Code; +import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage; import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.DenseVectorMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.ExternalHandlerQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.FromMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.NearestNeighbourQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultInfoMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultTupleMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultsMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.SubExpressionQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.VectorMessage; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.DistanceElement; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; -import java.util.*; - public class ADAMproStreamingSelector extends AbstractADAMproSelector { private static final Logger LOGGER = LogManager.getLogger(); @@ -28,7 +44,7 @@ private List> executeQuery(QueryMessage qm) { ArrayList resultList = this.adampro .streamingStandardQuery(qm); - if (resultList.isEmpty()){ + if (resultList.isEmpty()) { return Collections.emptyList(); } @@ -62,23 +78,23 @@ public List getNearestNeighboursGeneric(int k, fl String column, Class distanceElementClass, ReadableQueryConfig config) { NearestNeighbourQueryMessage nnqMessage = mb.buildNearestNeighbourQueryMessage(column, - DataMessageConverter.convertVectorMessage(vector), k, config); + DataMessageConverter.convertVectorMessage(vector), k, config); QueryMessage sqMessage = this.mb.buildQueryMessage(config.getHints().isEmpty() ? ADAMproMessageBuilder.DEFAULT_HINT : config.getHints(), fromMessage, this.mb.inList("id", config.getRelevantSegmentIds()), ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage); ArrayList resultList = this.adampro .streamingStandardQuery(sqMessage); - if (resultList.isEmpty()){ + if (resultList.isEmpty()) { return Collections.emptyList(); } List _return = new ArrayList<>(k); - for (QueryResultsMessage result : resultList){ + for (QueryResultsMessage result : resultList) { AckMessage ack = result.getAck(); if (ack.getCode() != AckMessage.Code.OK) { LOGGER.error("error in getNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - LOGGER.error("Query was {} ",sqMessage.toString()); + LOGGER.error("Query was {} ", sqMessage.toString()); continue; } @@ -98,22 +114,22 @@ public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { - if(vectors == null || vectors.isEmpty()){ + if (vectors == null || vectors.isEmpty()) { return Collections.emptyList(); } List messages = new ArrayList<>(vectors.size()); - for(int i = 0; i < vectors.size(); ++i) { + for (int i = 0; i < vectors.size(); ++i) { float[] vector = vectors.get(i); ReadableQueryConfig config = configs.get(i); NearestNeighbourQueryMessage nnqMessage = mb.buildNearestNeighbourQueryMessage(column, DataMessageConverter.convertVectorMessage(vector), k, config); QueryMessage sqMessage = this.mb - .buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fromMessage, - this.mb.inList("id", config.getRelevantSegmentIds()), - ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage); + .buildQueryMessage(ADAMproMessageBuilder.DEFAULT_HINT, fromMessage, + this.mb.inList("id", config.getRelevantSegmentIds()), + ADAMproMessageBuilder.DEFAULT_PROJECTION_MESSAGE, nnqMessage); messages.add(sqMessage); @@ -122,13 +138,13 @@ public List getBatchedNearestNeighbours(int k, ArrayList resultList = this.adampro .streamingStandardQuery(messages); - if (resultList.isEmpty()){ + if (resultList.isEmpty()) { return Collections.emptyList(); } List _return = new ArrayList<>(k); - for (QueryResultsMessage result : resultList){ + for (QueryResultsMessage result : resultList) { AckMessage ack = result.getAck(); if (ack.getCode() != AckMessage.Code.OK) { LOGGER.error("error in getBatchedNearestNeighbours on entity {}, ({}) : {}", entityName, ack.getCode(), ack.getMessage()); @@ -167,7 +183,7 @@ public List> getNearestNeighbourRows(int k, f ArrayList> _return = new ArrayList<>(k); - for(QueryResultsMessage result: resultList) { + for (QueryResultsMessage result : resultList) { if (result.getResponsesCount() == 0) { continue; @@ -180,7 +196,7 @@ public List> getNearestNeighbourRows(int k, f if (ack.getCode() != Code.OK) { LOGGER.error("error in getNearestNeighbourRows, entity {} ({}) : {}", entityName, ack.getCode(), ack.getMessage()); - continue; + continue; } _return.addAll(resultsToMap(response.getResultsList())); } @@ -194,8 +210,7 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v ArrayList resultList = this.adampro.streamingStandardQuery(qbqm); ArrayList _return = new ArrayList<>(); - - for(QueryResultsMessage r : resultList) { + for (QueryResultsMessage r : resultList) { if (r.getResponsesCount() == 0) { continue; @@ -265,7 +280,6 @@ public List> getFulltextRows(int rows, String final Map parameters = new HashMap<>(); parameters.put("rows", Integer.toString(rows)); - final StringBuilder sb = new StringBuilder(); sb.append('('); for (String item : terms) { @@ -307,5 +321,4 @@ private List> executeBooleanQuery(BooleanQuer } - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java index 222a64557..099f68f1d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java @@ -5,22 +5,37 @@ import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.adampro.grpc.AdamDefinitionGrpc; import org.vitrivr.adampro.grpc.AdamDefinitionGrpc.AdamDefinitionFutureStub; import org.vitrivr.adampro.grpc.AdamGrpc; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.BatchedQueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.CreateEntityMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.EmptyMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.EntityNameMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.EntityPropertiesMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.ExistsMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.IndexNameMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.InsertMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.PreviewMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.PropertiesMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultsMessage; import org.vitrivr.adampro.grpc.AdamSearchGrpc; import org.vitrivr.adampro.grpc.AdamSearchGrpc.AdamSearchFutureStub; import org.vitrivr.adampro.grpc.AdamSearchGrpc.AdamSearchStub; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.util.LogHelper; -import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.*; - public class ADAMproWrapper implements AutoCloseable { private static final Logger LOGGER = LogManager.getLogger(); @@ -40,7 +55,7 @@ public class ADAMproWrapper implements AutoCloseable { private static final int maxMessageSize = 10_000_000; private static final long maxCallTimeOutMs = 300_000; //TODO expose to config - + public ADAMproWrapper(DatabaseConfig config) { NettyChannelBuilder builder = NettyChannelBuilder.forAddress(config.getHost(), config.getPort()).maxInboundMessageSize(maxMessageSize); if (config.getPlaintext()) { @@ -98,7 +113,7 @@ public ListenableFuture existsEntity(String eName) { return this.definitionStub.existsEntity( EntityNameMessage.newBuilder().setEntity(eName).build()); } - + public boolean existsEntityBlocking(String eName) { ListenableFuture future = existsEntity(eName); try { @@ -123,7 +138,7 @@ public ListenableFuture booleanQuery(QueryMessage message) return Futures.withTimeout(standardQuery(message), maxCallTimeOutMs, TimeUnit.MILLISECONDS, timeoutService); } - public ListenableFuture standardQuery(QueryMessage message) { + public ListenableFuture standardQuery(QueryMessage message) { synchronized (this.searchStub) { return Futures.withTimeout(this.searchStub.doQuery(message), maxCallTimeOutMs, TimeUnit.MILLISECONDS, timeoutService); } @@ -173,7 +188,7 @@ public void onCompleted() { public ArrayList streamingStandardQuery(Collection messages) { - if(messages == null || messages.isEmpty()){ + if (messages == null || messages.isEmpty()) { return new ArrayList<>(0); } @@ -204,7 +219,7 @@ public void onCompleted() { } catch (InterruptedException e) { //ignore } - for(QueryMessage message : messages){ + for (QueryMessage message : messages) { queryMessageStreamObserver.onNext(message); } queryMessageStreamObserver.onCompleted(); @@ -258,7 +273,7 @@ public void close() { } - public ListenableFuture dropEntity(String entityName){ + public ListenableFuture dropEntity(String entityName) { return this.definitionStub.dropEntity(EntityNameMessage.newBuilder().setEntity(entityName).build()); } @@ -272,11 +287,11 @@ public boolean dropEntityBlocking(String entityName) { } } - public ListenableFuture dropIndex(String indexName){ + public ListenableFuture dropIndex(String indexName) { return this.definitionStub.dropIndex(IndexNameMessage.newBuilder().setIndex(indexName).build()); } - public boolean dropIndexBlocking(String indexName){ + public boolean dropIndexBlocking(String indexName) { ListenableFuture future = this.dropIndex(indexName); try { return future.get().getCode() == AckMessage.Code.OK; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java index 94413287e..3f9ae759c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java @@ -1,20 +1,25 @@ package org.vitrivr.cineast.core.db.adampro; import com.google.common.util.concurrent.ListenableFuture; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.adampro.grpc.AdamGrpc; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage.Code; +import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage; import org.vitrivr.adampro.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.FromMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.InsertMessage; import org.vitrivr.adampro.grpc.AdamGrpc.InsertMessage.TupleInsertMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultInfoMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultsMessage; import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.util.LogHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; - public class ADAMproWriter extends ProtobufTupleGenerator { private static final Logger LOGGER = LogManager.getLogger(); @@ -26,7 +31,7 @@ public class ADAMproWriter extends ProtobufTupleGenerator { private FromMessage from; private final ADAMproWrapper adampro; - public ADAMproWriter(ADAMproWrapper wrapper){ + public ADAMproWriter(ADAMproWrapper wrapper) { this.adampro = wrapper; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java index 091837274..1d8e6d081 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java @@ -3,8 +3,20 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import com.google.common.util.concurrent.ListenableFuture; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import org.vitrivr.adampro.grpc.AdamGrpc.AckMessage.Code; -import org.vitrivr.adampro.grpc.AdamGrpc.*; +import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.ExistsMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.FromMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.PreviewMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultInfoMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultTupleMessage; +import org.vitrivr.adampro.grpc.AdamGrpc.QueryResultsMessage; import org.vitrivr.cineast.core.data.DefaultValueHashMap; import org.vitrivr.cineast.core.data.distance.DistanceElement; import org.vitrivr.cineast.core.data.providers.primitive.NothingProvider; @@ -13,13 +25,6 @@ import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - public abstract class AbstractADAMproSelector implements DBSelector { /** @@ -39,7 +44,7 @@ public abstract class AbstractADAMproSelector implements DBSelector { final ADAMproWrapper adampro; - public AbstractADAMproSelector(ADAMproWrapper wrapper){ + public AbstractADAMproSelector(ADAMproWrapper wrapper) { this.adampro = wrapper; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ProtobufTupleGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ProtobufTupleGenerator.java index 6eb15cdae..2f4db8cd1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ProtobufTupleGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ProtobufTupleGenerator.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.db.adampro; +import java.util.HashMap; import org.vitrivr.adampro.grpc.AdamGrpc; import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; import org.vitrivr.adampro.grpc.AdamGrpc.DenseVectorMessage; @@ -13,98 +14,96 @@ import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; -import java.util.HashMap; - public abstract class ProtobufTupleGenerator extends AbstractPersistencyWriter { - private static final Builder builder = AdamGrpc.InsertMessage.TupleInsertMessage.newBuilder(); - - private final DataMessage.Builder insertMessageBuilder = DataMessage.newBuilder(); - - private DataMessage generateInsertMessage(Object o){ - synchronized(insertMessageBuilder){ - insertMessageBuilder.clear(); - if(o instanceof Long){ - return insertMessageBuilder.setLongData((Long)o).build(); - } - if(o instanceof Integer){ - return insertMessageBuilder.setIntData((Integer)o).build(); - } - if(o instanceof Float){ - return insertMessageBuilder.setFloatData((Float)o).build(); - } - if(o instanceof Double){ - return insertMessageBuilder.setDoubleData((Double)o).build(); - } - if(o instanceof Boolean){ - return insertMessageBuilder.setBooleanData((Boolean)o).build(); - } - if(o instanceof String){ - return insertMessageBuilder.setStringData((String)o).build(); - } - if(o instanceof float[]){ - return insertMessageBuilder.setVectorData(generateVectorMessage(new FloatArrayIterable((float[])o))).build(); - } - if(o instanceof int[]){ - return insertMessageBuilder.setVectorData(generateIntVectorMessage(new IntArrayIterable((int[])o))).build(); - } - if(o instanceof ReadableFloatVector){ - return insertMessageBuilder.setVectorData(generateVectorMessage(ReadableFloatVector.toList((ReadableFloatVector) o))).build(); - } - if(o == null){ - return insertMessageBuilder.setStringData("null").build(); - } - return insertMessageBuilder.setStringData(o.toString()).build(); - } - } - - private final VectorMessage.Builder VectorMessageBuilder = VectorMessage.newBuilder(); - private final DenseVectorMessage.Builder denseVectorMessageBuilder = DenseVectorMessage.newBuilder(); - private final IntVectorMessage.Builder intVectorMessageBuilder = IntVectorMessage.newBuilder(); - - private VectorMessage generateVectorMessage(Iterable vector){ - synchronized (VectorMessageBuilder) { - VectorMessageBuilder.clear(); - DenseVectorMessage msg; - synchronized (denseVectorMessageBuilder) { - denseVectorMessageBuilder.clear(); - msg = denseVectorMessageBuilder.addAllVector(vector).build(); - } - return VectorMessageBuilder.setDenseVector(msg).build(); - } - } - - private VectorMessage generateIntVectorMessage(Iterable vector){ - synchronized (VectorMessageBuilder) { - VectorMessageBuilder.clear(); - IntVectorMessage msg; - synchronized (intVectorMessageBuilder) { - intVectorMessageBuilder.clear(); - msg = intVectorMessageBuilder.addAllVector(vector).build(); - } - return VectorMessageBuilder.setIntVector(msg).build(); - } - } - - protected ProtobufTupleGenerator(String...names){ - super(names); - } - - protected ProtobufTupleGenerator(){ - super(); - } + private static final Builder builder = AdamGrpc.InsertMessage.TupleInsertMessage.newBuilder(); + + private final DataMessage.Builder insertMessageBuilder = DataMessage.newBuilder(); + + private DataMessage generateInsertMessage(Object o) { + synchronized (insertMessageBuilder) { + insertMessageBuilder.clear(); + if (o instanceof Long) { + return insertMessageBuilder.setLongData((Long) o).build(); + } + if (o instanceof Integer) { + return insertMessageBuilder.setIntData((Integer) o).build(); + } + if (o instanceof Float) { + return insertMessageBuilder.setFloatData((Float) o).build(); + } + if (o instanceof Double) { + return insertMessageBuilder.setDoubleData((Double) o).build(); + } + if (o instanceof Boolean) { + return insertMessageBuilder.setBooleanData((Boolean) o).build(); + } + if (o instanceof String) { + return insertMessageBuilder.setStringData((String) o).build(); + } + if (o instanceof float[]) { + return insertMessageBuilder.setVectorData(generateVectorMessage(new FloatArrayIterable((float[]) o))).build(); + } + if (o instanceof int[]) { + return insertMessageBuilder.setVectorData(generateIntVectorMessage(new IntArrayIterable((int[]) o))).build(); + } + if (o instanceof ReadableFloatVector) { + return insertMessageBuilder.setVectorData(generateVectorMessage(ReadableFloatVector.toList((ReadableFloatVector) o))).build(); + } + if (o == null) { + return insertMessageBuilder.setStringData("null").build(); + } + return insertMessageBuilder.setStringData(o.toString()).build(); + } + } - @Override + private final VectorMessage.Builder VectorMessageBuilder = VectorMessage.newBuilder(); + private final DenseVectorMessage.Builder denseVectorMessageBuilder = DenseVectorMessage.newBuilder(); + private final IntVectorMessage.Builder intVectorMessageBuilder = IntVectorMessage.newBuilder(); + + private VectorMessage generateVectorMessage(Iterable vector) { + synchronized (VectorMessageBuilder) { + VectorMessageBuilder.clear(); + DenseVectorMessage msg; + synchronized (denseVectorMessageBuilder) { + denseVectorMessageBuilder.clear(); + msg = denseVectorMessageBuilder.addAllVector(vector).build(); + } + return VectorMessageBuilder.setDenseVector(msg).build(); + } + } + + private VectorMessage generateIntVectorMessage(Iterable vector) { + synchronized (VectorMessageBuilder) { + VectorMessageBuilder.clear(); + IntVectorMessage msg; + synchronized (intVectorMessageBuilder) { + intVectorMessageBuilder.clear(); + msg = intVectorMessageBuilder.addAllVector(vector).build(); + } + return VectorMessageBuilder.setIntVector(msg).build(); + } + } + + protected ProtobufTupleGenerator(String... names) { + super(names); + } + + protected ProtobufTupleGenerator() { + super(); + } + + @Override public TupleInsertMessage getPersistentRepresentation(PersistentTuple tuple) { synchronized (builder) { - builder.clear(); + builder.clear(); HashMap tmpMap = new HashMap<>(); int nameIndex = 0; - - for(Object o : tuple.getElements()){ - + + for (Object o : tuple.getElements()) { + tmpMap.put(names[nameIndex++], generateInsertMessage(o)); - + } return builder.putAllData(tmpMap).build(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 3c7825875..47c11e144 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -7,19 +7,10 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import io.grpc.StatusRuntimeException; -import java.util.Objects; -import java.util.Optional; -import org.vitrivr.cottontail.client.TupleIterator; -import org.vitrivr.cottontail.client.language.basics.Constants; -import org.vitrivr.cottontail.client.language.basics.Type; -import org.vitrivr.cottontail.client.language.ddl.AboutEntity; -import org.vitrivr.cottontail.client.language.ddl.CreateEntity; -import org.vitrivr.cottontail.client.language.ddl.CreateIndex; -import org.vitrivr.cottontail.client.language.ddl.CreateSchema; -import org.vitrivr.cottontail.client.language.ddl.DropEntity; - import java.util.Arrays; import java.util.HashMap; +import java.util.Objects; +import java.util.Optional; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; @@ -28,15 +19,25 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.language.basics.Constants; +import org.vitrivr.cottontail.client.language.basics.Type; +import org.vitrivr.cottontail.client.language.ddl.AboutEntity; +import org.vitrivr.cottontail.client.language.ddl.CreateEntity; +import org.vitrivr.cottontail.client.language.ddl.CreateIndex; +import org.vitrivr.cottontail.client.language.ddl.CreateSchema; +import org.vitrivr.cottontail.client.language.ddl.DropEntity; import org.vitrivr.cottontail.client.language.ddl.ListSchemas; import org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType; public final class CottontailEntityCreator implements EntityCreator { public static final String COTTONTAIL_PREFIX = "cottontail"; - public static final String INDEX_HINT = COTTONTAIL_PREFIX+".index"; + public static final String INDEX_HINT = COTTONTAIL_PREFIX + ".index"; - /** Internal reference to the {@link CottontailWrapper} used by this {@link CottontailEntityCreator}. */ + /** + * Internal reference to the {@link CottontailWrapper} used by this {@link CottontailEntityCreator}. + */ private final CottontailWrapper cottontail; public CottontailEntityCreator(DatabaseConfig config) { @@ -255,7 +256,7 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d } // TODO (LS, 18.11.2020) Shouldn't we also have abstract indices in the db abstraction layer? final Optional hint = attribute.getHint(INDEX_HINT); - if (hint.isPresent()){ + if (hint.isPresent()) { IndexType idx = IndexType.valueOf(hint.get()); this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), idx, txId); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 071f180f5..92b39465d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -1,15 +1,13 @@ package org.vitrivr.cineast.core.db.cottontaildb; -import org.apache.commons.lang3.time.StopWatch; -import org.vitrivr.cottontail.client.stub.SimpleClient; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; - import java.util.concurrent.TimeUnit; - +import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cottontail.client.stub.SimpleClient; public final class CottontailWrapper implements AutoCloseable { @@ -30,8 +28,7 @@ public final class CottontailWrapper implements AutoCloseable { public final SimpleClient client; /** - * Flag indicating that his {@link CottontailWrapper}'s {@link ManagedChannel} should be kept - * open. + * Flag indicating that his {@link CottontailWrapper}'s {@link ManagedChannel} should be kept open. */ public final boolean keepOpen; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index 9c38b8334..eb5a16812 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -3,7 +3,6 @@ import io.grpc.StatusRuntimeException; import java.util.List; -import java.util.logging.Logger; import org.apache.logging.log4j.LogManager; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/MetadataAccessSpecification.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/MetadataAccessSpecification.java index 7c80ef38b..243b66d42 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/MetadataAccessSpecification.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/MetadataAccessSpecification.java @@ -7,16 +7,16 @@ */ public class MetadataAccessSpecification { - public final MetadataType type; - public final String domain; - public final String key; + public final MetadataType type; + public final String domain; + public final String key; - public MetadataAccessSpecification( - @JsonProperty("type") MetadataType type, - @JsonProperty("domain") String domain, - @JsonProperty("key") String key) { - this.type = type; - this.domain = domain; - this.key = key; - } + public MetadataAccessSpecification( + @JsonProperty("type") MetadataType type, + @JsonProperty("domain") String domain, + @JsonProperty("key") String key) { + this.type = type; + this.domain = domain; + this.key = key; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java index 4a1ced9a8..29a4ea117 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java @@ -1,31 +1,32 @@ package org.vitrivr.cineast.core.db.dao.reader; -import org.vitrivr.cineast.core.db.DBSelector; - import java.io.Closeable; +import org.vitrivr.cineast.core.db.DBSelector; public abstract class AbstractEntityReader implements Closeable { - /** DBSelector instance used to perform the DB lookup. */ - protected final DBSelector selector; - - /** - * Constructor for AbstractEntityReader - * - * @param selector DBSelector to use for the MediaObjectMetadataReader instance. - */ - public AbstractEntityReader(DBSelector selector) { - this.selector = selector; - } - - /** - * Closes the selector, relinquishing associated resources. - */ - @Override - public void close(){ - this.selector.close(); - } + /** + * DBSelector instance used to perform the DB lookup. + */ + protected final DBSelector selector; + + /** + * Constructor for AbstractEntityReader + * + * @param selector DBSelector to use for the MediaObjectMetadataReader instance. + */ + public AbstractEntityReader(DBSelector selector) { + this.selector = selector; + } + + /** + * Closes the selector, relinquishing associated resources. + */ + @Override + public void close() { + this.selector.close(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java index cc203ab3e..c8a30918c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java @@ -1,16 +1,13 @@ package org.vitrivr.cineast.core.db.dao.reader; import com.google.common.collect.Lists; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.commons.lang3.time.StopWatch; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; @@ -19,7 +16,6 @@ import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; import org.vitrivr.cineast.core.db.dao.MetadataType; -import org.vitrivr.cineast.core.util.TimeHelper; /** * Abstraction layer for segment and object metadata retrieval. @@ -28,119 +24,119 @@ */ public abstract class AbstractMetadataReader extends AbstractEntityReader { - private static final Logger LOGGER = LogManager.getLogger(); - private final String tableName; - private final String idColName; - - public AbstractMetadataReader(DBSelector selector, String tableName, String idColName) { - super(selector); - this.tableName = tableName; - this.idColName = idColName; - this.selector.open(tableName); - } - - abstract R resultToDescriptor(Map result) throws DatabaseLookupException; - - public List lookupMultimediaMetadata(String id) { - return this.lookupMultimediaMetadata(Lists.newArrayList(id)); - } - - public List lookupMultimediaMetadata(List ids) { - StopWatch watch = StopWatch.createStarted(); - ids = sanitizeIds(ids); - LOGGER.trace("Loading metadata for {} elements", ids.size()); - final List> results = this.selector.getRows(idColName, ids); - if (results.isEmpty()) { - LOGGER.debug("Could not find any metadata for provided IDs, Excerpt: {}. ID count: {}", String.join(", ", ids.subList(0, Math.min(5, ids.size()))), ids.size()); - } - List list = mapToResultList(results); - watch.stop(); - LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), list.size()); - return list; - } - - public List findBySpec(List ids, List spec) { - if (ids == null || spec == null) { - LOGGER.warn("provided id-list {} or spec {} is null, returning empty list", ids, spec); - return new ArrayList<>(); - } - StopWatch watch = StopWatch.createStarted(); - ids = sanitizeIds(ids); - spec = sanitizeSpec(spec); - List> results = selector.getMetadataByIdAndSpec(ids, spec, idColName); - LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), results.size()); - return mapToResultList(results); - } - - public List findBySpec(List spec) { - StopWatch watch = StopWatch.createStarted(); - spec = sanitizeSpec(spec); - List> results = selector.getMetadataBySpec(spec); - LOGGER.debug("Performed metadata lookup in {} ms. {} results.", watch.getTime(TimeUnit.MILLISECONDS), results.size()); - return mapToResultList(results); + private static final Logger LOGGER = LogManager.getLogger(); + private final String tableName; + private final String idColName; + + public AbstractMetadataReader(DBSelector selector, String tableName, String idColName) { + super(selector); + this.tableName = tableName; + this.idColName = idColName; + this.selector.open(tableName); + } + + abstract R resultToDescriptor(Map result) throws DatabaseLookupException; + + public List lookupMultimediaMetadata(String id) { + return this.lookupMultimediaMetadata(Lists.newArrayList(id)); + } + + public List lookupMultimediaMetadata(List ids) { + StopWatch watch = StopWatch.createStarted(); + ids = sanitizeIds(ids); + LOGGER.trace("Loading metadata for {} elements", ids.size()); + final List> results = this.selector.getRows(idColName, ids); + if (results.isEmpty()) { + LOGGER.debug("Could not find any metadata for provided IDs, Excerpt: {}. ID count: {}", String.join(", ", ids.subList(0, Math.min(5, ids.size()))), ids.size()); } - - public List findBySpec(String id, MetadataAccessSpecification spec) { - return this.findBySpec(id, Lists.newArrayList(spec)); + List list = mapToResultList(results); + watch.stop(); + LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), list.size()); + return list; + } + + public List findBySpec(List ids, List spec) { + if (ids == null || spec == null) { + LOGGER.warn("provided id-list {} or spec {} is null, returning empty list", ids, spec); + return new ArrayList<>(); } - - public List findBySpec(String id, List spec) { - if (id == null || id.isEmpty()) { - LOGGER.warn("Provided id is null or empty, returning empty list"); - return new ArrayList<>(); - } - return this.findBySpec(Lists.newArrayList(id), spec); + StopWatch watch = StopWatch.createStarted(); + ids = sanitizeIds(ids); + spec = sanitizeSpec(spec); + List> results = selector.getMetadataByIdAndSpec(ids, spec, idColName); + LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), results.size()); + return mapToResultList(results); + } + + public List findBySpec(List spec) { + StopWatch watch = StopWatch.createStarted(); + spec = sanitizeSpec(spec); + List> results = selector.getMetadataBySpec(spec); + LOGGER.debug("Performed metadata lookup in {} ms. {} results.", watch.getTime(TimeUnit.MILLISECONDS), results.size()); + return mapToResultList(results); + } + + public List findBySpec(String id, MetadataAccessSpecification spec) { + return this.findBySpec(id, Lists.newArrayList(spec)); + } + + public List findBySpec(String id, List spec) { + if (id == null || id.isEmpty()) { + LOGGER.warn("Provided id is null or empty, returning empty list"); + return new ArrayList<>(); } - - public List findBySpec(List ids, MetadataAccessSpecification spec) { - return this.findBySpec(ids, Lists.newArrayList(spec)); + return this.findBySpec(Lists.newArrayList(id), spec); + } + + public List findBySpec(List ids, MetadataAccessSpecification spec) { + return this.findBySpec(ids, Lists.newArrayList(spec)); + } + + public List findBySpec(MetadataAccessSpecification... spec) { + return this.findBySpec(Lists.newArrayList(spec)); + } + + public List findBySpec(MetadataAccessSpecification spec) { + return this.findBySpec(Lists.newArrayList(spec)); + } + + public List sanitizeSpec(List spec) { + // filter null objects + if (spec.stream().anyMatch(Objects::isNull)) { + LOGGER.warn("provided spec-list contains null elements which will be ignored"); + spec = spec.stream().filter(Objects::nonNull).collect(Collectors.toList()); } - - public List findBySpec(MetadataAccessSpecification... spec) { - return this.findBySpec(Lists.newArrayList(spec)); + // filter non-object specs if this is an object reader + if (Objects.equals(this.tableName, MediaObjectMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type != MetadataType.OBJECT)) { + LOGGER.trace("provided spec-list includes non-object tuples, but this is an object reader. These will be ignored."); + spec = spec.stream().filter(el -> el.type == MetadataType.OBJECT).collect(Collectors.toList()); } - - public List findBySpec(MetadataAccessSpecification spec) { - return this.findBySpec(Lists.newArrayList(spec)); + // filter non-segment specs if this is a segment reader + if (Objects.equals(this.tableName, MediaSegmentMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type != MetadataType.SEGMENT)) { + LOGGER.trace("provided spec-list includes non-segment tuples, but this is a segment reader. These will be ignored."); + spec = spec.stream().filter(el -> el.type == MetadataType.SEGMENT).collect(Collectors.toList()); } - - public List sanitizeSpec(List spec) { - // filter null objects - if (spec.stream().anyMatch(Objects::isNull)) { - LOGGER.warn("provided spec-list contains null elements which will be ignored"); - spec = spec.stream().filter(Objects::nonNull).collect(Collectors.toList()); - } - // filter non-object specs if this is an object reader - if (Objects.equals(this.tableName, MediaObjectMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type != MetadataType.OBJECT)) { - LOGGER.trace("provided spec-list includes non-object tuples, but this is an object reader. These will be ignored."); - spec = spec.stream().filter(el -> el.type == MetadataType.OBJECT).collect(Collectors.toList()); - } - // filter non-segment specs if this is a segment reader - if (Objects.equals(this.tableName, MediaSegmentMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type != MetadataType.SEGMENT)) { - LOGGER.trace("provided spec-list includes non-segment tuples, but this is a segment reader. These will be ignored."); - spec = spec.stream().filter(el -> el.type == MetadataType.SEGMENT).collect(Collectors.toList()); - } - return spec; - } - - public List mapToResultList(List> results) { - final ArrayList list = new ArrayList<>(results.size()); - results.forEach(r -> { - try { - list.add(resultToDescriptor(r)); - } catch (DatabaseLookupException exception) { - LOGGER.fatal("Could not map data. This is a programmer's error!"); - } - }); - return list; - } - - public static List sanitizeIds(List ids) { - if (ids.stream().anyMatch(el -> el == null || el.isEmpty())) { - LOGGER.warn("provided id-list contains null or empty elements which will be ignored"); - ids = ids.stream().filter(el -> el != null && !el.isEmpty()).collect(Collectors.toList()); - } - return ids; + return spec; + } + + public List mapToResultList(List> results) { + final ArrayList list = new ArrayList<>(results.size()); + results.forEach(r -> { + try { + list.add(resultToDescriptor(r)); + } catch (DatabaseLookupException exception) { + LOGGER.fatal("Could not map data. This is a programmer's error!"); + } + }); + return list; + } + + public static List sanitizeIds(List ids) { + if (ids.stream().anyMatch(el -> el == null || el.isEmpty())) { + LOGGER.warn("provided id-list contains null or empty elements which will be ignored"); + ids = ids.stream().filter(el -> el != null && !el.isEmpty()).collect(Collectors.toList()); } + return ids; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java index 7632856a0..4f0288110 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java @@ -2,7 +2,6 @@ import com.google.common.collect.Lists; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java index c036f1d1d..367e9a9d4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java @@ -1,24 +1,26 @@ package org.vitrivr.cineast.core.db.dao.reader; +import static org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor.FIELDNAMES; + import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.data.providers.primitive.StringProvider; -import org.vitrivr.cineast.core.data.providers.primitive.StringProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; import org.vitrivr.cineast.core.db.DBSelector; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor.FIELDNAMES; - public class MediaSegmentReader extends AbstractEntityReader { private static final Logger LOGGER = LogManager.getLogger(); @@ -45,7 +47,7 @@ private static Optional propertiesToDescriptor( && properties.containsKey(FIELDNAMES[6])) { return Optional.of( - new MediaSegmentDescriptor( + new MediaSegmentDescriptor( properties.get(FIELDNAMES[1]).getString(), properties.get(FIELDNAMES[0]).getString(), properties.get(FIELDNAMES[2]).getInt(), @@ -87,12 +89,12 @@ public ListMultimap lookUpSegmentsOfObjects( return Multimaps.index(descriptors.iterator(), MediaSegmentDescriptor::getObjectId); } - public List lookUpSegmentByNumber(String objectId, int segmentNumber){ + public List lookUpSegmentByNumber(String objectId, int segmentNumber) { List all = this.lookUpSegmentsOfObject(objectId); return all.stream().filter(it -> it.getSequenceNumber() == segmentNumber).collect(Collectors.toList()); } - public List lookUpSegmentsByNumberRange(String objectId, int lower, int upper){ //TODO implementing this without selecting all segments would require additional functionality in DBSelector + public List lookUpSegmentsByNumberRange(String objectId, int lower, int upper) { //TODO implementing this without selecting all segments would require additional functionality in DBSelector List all = this.lookUpSegmentsOfObject(objectId); return all.stream().filter(it -> it.getSequenceNumber() >= lower && it.getSequenceNumber() <= upper).collect(Collectors.toList()); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java index 0b0aba6cc..673db91af 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java @@ -1,94 +1,97 @@ package org.vitrivr.cineast.core.db.dao.writer; import java.io.Closeable; -import org.vitrivr.cineast.core.db.PersistencyWriter; -import org.vitrivr.cineast.core.db.PersistentTuple; - import java.util.ArrayList; import java.util.List; import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; +import org.vitrivr.cineast.core.db.PersistencyWriter; +import org.vitrivr.cineast.core.db.PersistentTuple; public abstract class AbstractBatchedEntityWriter implements Closeable { - /** The {@link Queue} used to store {@link PersistentTuple}s until they are flushed to disk. */ - private final ArrayBlockingQueue buffer; - - /** {@link PersistencyWriter} instance used to persist changes to the underlying persistence layer. */ - protected PersistencyWriter writer; - - private final boolean batch; - - protected AbstractBatchedEntityWriter(PersistencyWriter writer, int batchsize, boolean init) { - this.batch = batchsize > 1; - if(this.batch){ - this.buffer = new ArrayBlockingQueue<>(batchsize); - } else { - this.buffer = null; //not used - } - this.writer = writer; - if (init) { - this.init(); - } - } - protected abstract void init(); - - protected abstract PersistentTuple generateTuple(T entity); - - /** - * Persists the provided entity by first converting it to a {@link PersistentTuple} and subsequently writing that tuple to the local buffer. - * If the buffer is full, i.e. the batch size was reached, then buffer is flushed first. - * - * @param entity The entity that should be persisted. - */ - public void write(T entity) { - final PersistentTuple tuple = this.generateTuple(entity); - if(tuple == null){ - return; // One of the entity's value provider was a NothingProvider, hence nothing is written. - } - if(this.batch) { - if (this.buffer.remainingCapacity() == 0) { - this.flush(); - } - this.buffer.offer(tuple); - } else { - this.writer.persist(tuple); - } - } + /** + * The {@link Queue} used to store {@link PersistentTuple}s until they are flushed to disk. + */ + private final ArrayBlockingQueue buffer; - public void write(List entity) { - entity.forEach(this::write); - } + /** + * {@link PersistencyWriter} instance used to persist changes to the underlying persistence layer. + */ + protected PersistencyWriter writer; - /** - * Drains the content of the buffer and writes it to the underlying persistence layer using the local {@link PersistencyWriter} instance. - */ - public final void flush() { - if(!this.batch){ - return; - } - final List batch = new ArrayList<>(buffer.size()); - this.buffer.drainTo(batch); - this.writer.persist(batch); + private final boolean batch; + + protected AbstractBatchedEntityWriter(PersistencyWriter writer, int batchsize, boolean init) { + this.batch = batchsize > 1; + if (this.batch) { + this.buffer = new ArrayBlockingQueue<>(batchsize); + } else { + this.buffer = null; //not used + } + this.writer = writer; + if (init) { + this.init(); } + } + + protected abstract void init(); + + protected abstract PersistentTuple generateTuple(T entity); - /** - * Flushes the buffer and closes the local {@link PersistencyWriter}. - */ - @Override - public final void close() { - if (this.writer != null) { - if (this.batch && this.buffer.size() > 0) { - this.flush(); - } - - if (this.writer != null) { - this.writer.close(); - this.writer = null; - } - } + /** + * Persists the provided entity by first converting it to a {@link PersistentTuple} and subsequently writing that tuple to the local buffer. If the buffer is full, i.e. the batch size was reached, then buffer is flushed first. + * + * @param entity The entity that should be persisted. + */ + public void write(T entity) { + final PersistentTuple tuple = this.generateTuple(entity); + if (tuple == null) { + return; // One of the entity's value provider was a NothingProvider, hence nothing is written. + } + if (this.batch) { + if (this.buffer.remainingCapacity() == 0) { + this.flush(); + } + this.buffer.offer(tuple); + } else { + this.writer.persist(tuple); + } + } + + public void write(List entity) { + entity.forEach(this::write); + } + + /** + * Drains the content of the buffer and writes it to the underlying persistence layer using the local {@link PersistencyWriter} instance. + */ + public final void flush() { + if (!this.batch) { + return; + } + final List batch = new ArrayList<>(buffer.size()); + this.buffer.drainTo(batch); + this.writer.persist(batch); + } + + /** + * Flushes the buffer and closes the local {@link PersistencyWriter}. + */ + @Override + public final void close() { + if (this.writer != null) { + if (this.batch && this.buffer.size() > 0) { + this.flush(); + } + + if (this.writer != null) { + this.writer.close(); + this.writer = null; + } } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java index f5c8eecd4..d42c88e75 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java @@ -30,8 +30,8 @@ public void init() { @Override protected PersistentTuple generateTuple(TagInstance entity) { float score = 1f; - if(entity.tag instanceof WeightedTag){ - score = ((WeightedTag)entity.tag).getWeight(); + if (entity.tag instanceof WeightedTag) { + score = ((WeightedTag) entity.tag).getWeight(); } return this.writer.generateTuple(entity.id, entity.tag.getId(), score); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java index 09e3e21e8..ac1f8fa0a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java @@ -5,23 +5,23 @@ import org.vitrivr.cineast.core.db.PersistentTuple; - public class MediaObjectWriter extends AbstractBatchedEntityWriter { - public MediaObjectWriter(PersistencyWriter writer) { - super(writer, 1, true); - } - /** - * - */ - @Override - protected void init() { - this.writer.setFieldNames(MediaObjectDescriptor.FIELDNAMES); - this.writer.open(MediaObjectDescriptor.ENTITY); - } + public MediaObjectWriter(PersistencyWriter writer) { + super(writer, 1, true); + } + + /** + * + */ + @Override + protected void init() { + this.writer.setFieldNames(MediaObjectDescriptor.FIELDNAMES); + this.writer.open(MediaObjectDescriptor.ENTITY); + } - @Override - protected PersistentTuple generateTuple(MediaObjectDescriptor entity) { - return this.writer.generateTuple(entity.getObjectId(), entity.getMediatypeId(), entity.getName(), entity.getPath()); - } + @Override + protected PersistentTuple generateTuple(MediaObjectDescriptor entity) { + return this.writer.generateTuple(entity.getObjectId(), entity.getMediatypeId(), entity.getName(), entity.getPath()); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java index e872ede20..cdc07fcf8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java @@ -8,26 +8,26 @@ public class SimpleFeatureDescriptorWriter extends AbstractBatchedEntityWriter { - private final String entityname; - - public SimpleFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { - this(writer, entityname, 1); - } - - public SimpleFeatureDescriptorWriter(PersistencyWriter writer, String entityname, int batchsize) { - super(writer, batchsize, false); - this.entityname = entityname; - this.init(); - } - - @Override - public void init() { - this.writer.open(this.entityname); - } - - @Override - protected PersistentTuple generateTuple(SimpleFeatureDescriptor entity) { - float[] array = ReadableFloatVector.toArray(entity.getFeature()); - return this.writer.generateTuple(entity.getSegmentId(), array); - } + private final String entityname; + + public SimpleFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { + this(writer, entityname, 1); + } + + public SimpleFeatureDescriptorWriter(PersistencyWriter writer, String entityname, int batchsize) { + super(writer, batchsize, false); + this.entityname = entityname; + this.init(); + } + + @Override + public void init() { + this.writer.open(this.entityname); + } + + @Override + protected PersistentTuple generateTuple(SimpleFeatureDescriptor entity) { + float[] array = ReadableFloatVector.toArray(entity.getFeature()); + return this.writer.generateTuple(entity.getSegmentId(), array); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/TagWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/TagWriter.java index ce17af847..fb8525ba8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/TagWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/TagWriter.java @@ -1,55 +1,52 @@ package org.vitrivr.cineast.core.db.dao.writer; +import java.io.Closeable; import org.vitrivr.cineast.core.data.tag.Tag; -import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.dao.reader.TagReader; -import java.io.Closeable; - public class TagWriter implements Closeable { - private final PersistencyWriter writer; - - public TagWriter(PersistencyWriter writer) { - this.writer = writer; + private final PersistencyWriter writer; + public TagWriter(PersistencyWriter writer) { + this.writer = writer; - if (this.writer == null) { - throw new NullPointerException("writer cannot be null"); - } - - this.writer.open(TagReader.TAG_ENTITY_NAME); - this.writer.setFieldNames(TagReader.TAG_ID_COLUMNNAME, TagReader.TAG_NAME_COLUMNNAME, TagReader.TAG_DESCRIPTION_COLUMNNAME); - } - - /** - * Adds and persist a new {@link Tag} entry. - * - * @param id ID of the new entry. - * @param name Name of the new entry. - * @param description Description of the new entry. - * @return True on success, false otherwise. - */ - public boolean addTag(String id, String name, String description) { - return this.writer.persist(this.writer.generateTuple(id, name, description)); + if (this.writer == null) { + throw new NullPointerException("writer cannot be null"); } - /** - * Adds and persist a new {@link Tag} entry. - * - * @param tag {@link Tag} that should be added. - */ - public boolean addTag(Tag tag) { - if (tag == null) { - return false; - } - return addTag(tag.getId(), tag.getName(), tag.getDescription()); + this.writer.open(TagReader.TAG_ENTITY_NAME); + this.writer.setFieldNames(TagReader.TAG_ID_COLUMNNAME, TagReader.TAG_NAME_COLUMNNAME, TagReader.TAG_DESCRIPTION_COLUMNNAME); + } + + /** + * Adds and persist a new {@link Tag} entry. + * + * @param id ID of the new entry. + * @param name Name of the new entry. + * @param description Description of the new entry. + * @return True on success, false otherwise. + */ + public boolean addTag(String id, String name, String description) { + return this.writer.persist(this.writer.generateTuple(id, name, description)); + } + + /** + * Adds and persist a new {@link Tag} entry. + * + * @param tag {@link Tag} that should be added. + */ + public boolean addTag(Tag tag) { + if (tag == null) { + return false; } + return addTag(tag.getId(), tag.getName(), tag.getDescription()); + } - @Override - public void close() { - this.writer.close(); - } + @Override + public void close() { + this.writer.close(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java index 68dc91b50..4edb4207e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java @@ -4,28 +4,25 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; - -import org.vitrivr.cineast.core.data.ReadableFloatVector; -import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; -import org.vitrivr.cineast.core.db.PersistentTuple; - import java.io.File; import java.io.FileNotFoundException; import java.io.PrintWriter; import java.util.List; +import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; +import org.vitrivr.cineast.core.db.PersistentTuple; public class JsonFileWriter extends AbstractPersistencyWriter { - + private File baseFolder; private PrintWriter out; private boolean first = true; - + public JsonFileWriter(File baseFolder) { this.baseFolder = baseFolder; } - - + @Override public boolean open(String name) { baseFolder.mkdirs(); @@ -40,7 +37,7 @@ public boolean open(String name) { return false; } } - + @Override public synchronized void close() { if (out == null) { @@ -52,11 +49,11 @@ public synchronized void close() { out.close(); out = null; } - + @Override public boolean persist(PersistentTuple tuple) { synchronized (out) { - if(!this.first){ + if (!this.first) { this.out.println(','); } this.out.print(this.getPersistentRepresentation(tuple).toString()); @@ -65,9 +62,9 @@ public boolean persist(PersistentTuple tuple) { } return true; - + } - + @Override public boolean persist(List tuples) { boolean success = true; @@ -83,20 +80,20 @@ public boolean persist(List tuples) { public boolean idExists(String id) { return false; } - + @Override public boolean exists(String key, String value) { return false; } - + @Override public JsonObject getPersistentRepresentation(PersistentTuple tuple) { - + int nameIndex = 0; - + JsonObject _return = new JsonObject(); - + for (Object o : tuple.getElements()) { if (o instanceof float[]) { _return.add(names[nameIndex++], toArray((float[]) o)); @@ -121,10 +118,10 @@ public JsonObject getPersistentRepresentation(PersistentTuple tuple) { _return.add(names[nameIndex++], new JsonPrimitive(o.toString())); } } - + return _return; } - + private static JsonArray toArray(boolean[] arr) { JsonArray jarr = new JsonArray(); for (int i = 0; i < arr.length; ++i) { @@ -132,7 +129,7 @@ private static JsonArray toArray(boolean[] arr) { } return jarr; } - + private static JsonArray toArray(float[] arr) { JsonArray jarr = new JsonArray(); for (int i = 0; i < arr.length; ++i) { @@ -140,7 +137,7 @@ private static JsonArray toArray(float[] arr) { } return jarr; } - + private static JsonArray toArray(int[] arr) { JsonArray jarr = new JsonArray(); for (int i = 0; i < arr.length; ++i) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonSelector.java index b16c898d2..5379c961d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonSelector.java @@ -1,22 +1,20 @@ package org.vitrivr.cineast.core.db.json; +import java.io.File; +import java.io.IOException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.ImporterSelector; import org.vitrivr.cineast.core.importer.JsonObjectImporter; - -import java.io.File; -import java.io.IOException; - public class JsonSelector extends ImporterSelector { - public JsonSelector(File baseDirectory){ + public JsonSelector(File baseDirectory) { super(baseDirectory); } private static final Logger LOGGER = LogManager.getLogger(); - + @Override protected JsonObjectImporter newImporter(File f) { try { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryStore.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryStore.java index e9babee02..9421a2a31 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryStore.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryStore.java @@ -12,27 +12,29 @@ /** - * This is a very simple in-memory key-value store implementation. It revolves around {@link Entity} objects, that hold - * instance of {@link PersistentTuple}s. Obtaining such a {@link PersistentTuple} by key can be done in O(1). All other - * operations take at least linear time to complete. - * - * - * {@link InMemoryStore} can be used as a storage engine for Cineast. That is, there are implementations of - * {@link PersistencyWriter} and {@link DBSelector} for this store. + * This is a very simple in-memory key-value store implementation. It revolves around {@link Entity} objects, that hold instance of {@link PersistentTuple}s. Obtaining such a {@link PersistentTuple} by key can be done in O(1). All other operations take at least linear time to complete. + *

    + *

    + * {@link InMemoryStore} can be used as a storage engine for Cineast. That is, there are implementations of {@link PersistencyWriter} and {@link DBSelector} for this store. * * @see InMemoryWriter * @see InMemoryEntityCreator - * */ public class InMemoryStore { - /** Shared instance of {@link InMemoryStore}. */ + /** + * Shared instance of {@link InMemoryStore}. + */ private static final InMemoryStore SHARED_STORE = new InMemoryStore(); - /** List of {@link Entity} objects held by this {@link InMemoryStore}. */ - private final Map entities = new HashMap<>(); + /** + * List of {@link Entity} objects held by this {@link InMemoryStore}. + */ + private final Map entities = new HashMap<>(); - /** Stamped lock to mediate access to {@link InMemoryStore}. */ + /** + * Stamped lock to mediate access to {@link InMemoryStore}. + */ private final StampedLock storeLock = new StampedLock(); /** @@ -44,11 +46,11 @@ public static InMemoryStore sharedInMemoryStore() { return SHARED_STORE; } - public InMemoryStore() {} + public InMemoryStore() { + } /** - * Returns the {@link Entity} for the given name or an empty {@link Optional}, if that - * {@link Entity} doesn't exist. + * Returns the {@link Entity} for the given name or an empty {@link Optional}, if that {@link Entity} doesn't exist. * * @param name Name of the {@link Entity} to return. * @return An optional {@link Entity} @@ -78,10 +80,9 @@ public boolean hasEntity(String name) { } /** - * Creates and returns the {@link Entity} for the given name or an empty {@link Optional}, if that - * {@link Entity} already exists and hence wasn't created. + * Creates and returns the {@link Entity} for the given name or an empty {@link Optional}, if that {@link Entity} already exists and hence wasn't created. * - * @param name Name of the {@link Entity} to create. + * @param name Name of the {@link Entity} to create. * @param columns The list of columns to create. * @return An optional {@link Entity} */ @@ -123,17 +124,22 @@ public void dropAll() { /** * An individual {@link Entity} in the {@link InMemoryStore}. - * */ public class Entity implements Iterable { - /** The {@link java.util.Map} that holds all the data stored in this {@link org.vitrivr.cineast.core.db.memory.InMemoryStore.Entity}. */ + /** + * The {@link java.util.Map} that holds all the data stored in this {@link org.vitrivr.cineast.core.db.memory.InMemoryStore.Entity}. + */ private final Map store = new TreeMap<>(); - /** Name of the columns held by this {@link Entity}. */ + /** + * Name of the columns held by this {@link Entity}. + */ private final String[] columns; - /** Stamped lock to mediate access to {@link Entity}. */ + /** + * Stamped lock to mediate access to {@link Entity}. + */ private final StampedLock lock = new StampedLock(); /** @@ -160,7 +166,7 @@ public boolean put(String key, PersistentTuple value) { } else { return false; } - }finally { + } finally { this.lock.unlockWrite(stamp); } } @@ -208,12 +214,10 @@ public void truncate() { } /** - * Returns true if this {@link Entity} contains a {@link PersistentTuple} for the given key, and - * false otherwise. + * Returns true if this {@link Entity} contains a {@link PersistentTuple} for the given key, and false otherwise. * * @param key The key to look up. - * @return True if {@link Entity} contains a {@link PersistentTuple} for the given key, and - * false otherwise + * @return True if {@link Entity} contains a {@link PersistentTuple} for the given key, and false otherwise */ public boolean has(String key) { final long stamp = this.lock.readLock(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java index 1541bed39..3f4a17b77 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java @@ -13,7 +13,6 @@ * Implementation of a Cineast {@link org.vitrivr.cineast.core.db.PersistencyWriter} on top of the {@link InMemoryStore}. * * @see InMemoryStore - * */ public class InMemoryWriter extends AbstractPersistencyWriter { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java index 4c9337b12..f4380257b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/AttributeDefinition.java @@ -7,102 +7,108 @@ public final class AttributeDefinition { - public enum AttributeType { - UNKOWNAT, - AUTO, - LONG, - INT, - FLOAT, - DOUBLE, - STRING, - TEXT, - BOOLEAN, - VECTOR, - BITSET, - GEOMETRY, - GEOGRAPHY + public enum AttributeType { + UNKOWNAT, + AUTO, + LONG, + INT, + FLOAT, + DOUBLE, + STRING, + TEXT, + BOOLEAN, + VECTOR, + BITSET, + GEOMETRY, + GEOGRAPHY + } + + /** + * The name of the {@link AttributeDefinition}. + */ + private final String name; + + /** + * The {@link AttributeType} of the {@link AttributeDefinition}. + */ + private final AttributeType type; + + /** + * Hints to the underlying storage engine. Those hints are highly implementation specific! l. + */ + private final Map hints; + + private final int length; + + /** + * Constructor for {@link AttributeDefinition} + * + * @param name Name of the attribute. + * @param type Type of the attribute + */ + public AttributeDefinition(String name, AttributeType type) { + this(name, type, new HashMap<>()); + } + + public AttributeDefinition(String name, AttributeType type, int length) { + this(name, type, length, new HashMap<>()); + } + + /** + * Constructor for {@link AttributeDefinition} + * + * @param name Name of the attribute. + * @param type Type of the attribute. + * @param hints Hint to the storage engine regarding the handler. + */ + public AttributeDefinition(String name, AttributeType type, Map hints) { + this(name, type, -1, hints); + } + + public AttributeDefinition(String name, AttributeType type, int length, Map hints) { + this.name = name; + this.type = type; + this.hints = hints; + this.length = length; + } + + /** + * Getter for the name of the {@link AttributeDefinition}. + * + * @return Name of the attribute. + */ + public String getName() { + return this.name; + } + + /** + * Getter for the type of the {@link AttributeDefinition}. + * + * @return Name of the attribute. + */ + public AttributeType getType() { + return this.type; + } + + public boolean hasHint(String hint) { + return this.hints.containsKey(hint); + } + + public Optional getHint(String hint) { + return Optional.ofNullable(this.hints.get(hint)); + } + + public String getHintOrDefault(String hint, String defaultValue) { + return this.hints.getOrDefault(hint, defaultValue); + } + + public void ifHintPresent(String hint, Consumer consumer) { + if (this.hints.containsKey(hint)) { + consumer.accept(this.hints.get(hint)); } + } - /** The name of the {@link AttributeDefinition}. */ - private final String name; - - /**The {@link AttributeType} of the {@link AttributeDefinition}. */ - private final AttributeType type; - - /** Hints to the underlying storage engine. Those hints are highly implementation specific! l. */ - private final Map hints; - - private final int length; - - /** - * Constructor for {@link AttributeDefinition} - * - * @param name Name of the attribute. - * @param type Type of the attribute - */ - public AttributeDefinition(String name, AttributeType type) { - this(name, type, new HashMap<>()); - } - - public AttributeDefinition(String name, AttributeType type, int length) { - this(name, type, length, new HashMap<>()); - } - - /** - * Constructor for {@link AttributeDefinition} - * - * @param name Name of the attribute. - * @param type Type of the attribute. - * @param hints Hint to the storage engine regarding the handler. - */ - public AttributeDefinition(String name, AttributeType type, Map hints) { - this(name, type, -1, hints); - } - - public AttributeDefinition(String name, AttributeType type, int length, Map hints) { - this.name = name; - this.type = type; - this.hints = hints; - this.length = length; - } - - /** - * Getter for the name of the {@link AttributeDefinition}. - * - * @return Name of the attribute. - */ - public String getName() { - return this.name; - } - - /** - * Getter for the type of the {@link AttributeDefinition}. - * - * @return Name of the attribute. - */ - public AttributeType getType() { - return this.type; - } - - public boolean hasHint(String hint) { - return this.hints.containsKey(hint); - } - - public Optional getHint(String hint) { - return Optional.ofNullable(this.hints.get(hint)); - } - - public String getHintOrDefault(String hint, String defaultValue) { - return this.hints.getOrDefault(hint, defaultValue); - } - - public void ifHintPresent(String hint, Consumer consumer) { - if (this.hints.containsKey(hint)) { - consumer.accept(this.hints.get(hint)); - } - } - - public int getLength(){ - return this.length; - } + public int getLength() { + return this.length; + } } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityDefinition.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityDefinition.java index bebc5f68f..170d85b83 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityDefinition.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityDefinition.java @@ -10,198 +10,200 @@ */ public final class EntityDefinition { + /** + * The entity's name + */ + private final String entityName; + /** + * Its attributes + */ + private final List attributes; + /** + * Flag whether this entity will have at maximum one vector per segment + */ + private final boolean oneVectorPerSegment; + + /** + * Creates a new entity definition with given parameters + * + * @param entityName The name of the entity + * @param attributes A list of attributes + * @param oneVectorPerSegment Whether this entity will have one vector per segment at max + */ + private EntityDefinition(String entityName, List attributes, boolean oneVectorPerSegment) { + this.entityName = entityName; + this.attributes = attributes; + this.oneVectorPerSegment = oneVectorPerSegment; + } + + /** + * Returns the name of the entity + * + * @return The name of the entity + */ + public String getEntityName() { + return entityName; + } + + /** + * Returns an immutable list of attributes + * + * @return An immutable list of attributes + */ + public List getAttributes() { + return Collections.unmodifiableList(attributes); + } + + /** + * Whether this entity will have one vector per segment at max + * + * @return This entity will have one vector per segment at max + */ + public boolean isOneVectorPerSegment() { + return oneVectorPerSegment; + } + + @Override + public String toString() { + return "EntityDefinition{" + + "entityName='" + entityName + '\'' + + ", attributes=" + attributes + + ", oneVectorPerSegment=" + oneVectorPerSegment + + '}'; + } + + /** + * A builder for {@link EntityDefinition}s + */ + public static class EntityDefinitionBuilder { + + private String name; + private List attrs = new ArrayList<>(); + private boolean oneVectorPerSegment; + /** - * The entity's name + * Creates a new builder for a named entity + * + * @param name The name of the entity */ - private final String entityName; + public EntityDefinitionBuilder(String name) { + this.name = name; + } + /** - * Its attributes + * Creates a new builder from an entity defintiion, to modify said one + * + * @param def The defintiion template for this builder + * @return A new builder */ - private final List attributes; + public static EntityDefinitionBuilder from(EntityDefinition def) { + EntityDefinitionBuilder builder = new EntityDefinitionBuilder(def.getEntityName()); + builder.attrs.addAll(def.getAttributes()); + builder.oneVectorPerSegment = def.oneVectorPerSegment; + return builder; + } + + public void reset() { + name = ""; + attrs.clear(); + oneVectorPerSegment = false; + } + /** - * Flag whether this entity will have at maximum one vector per segment + * Sets the name of the entity to build + * + * @param name The name of the entity + * @return The builder */ - private final boolean oneVectorPerSegment; + public EntityDefinitionBuilder withName(String name) { + this.name = name; + return this; + } /** - * Creates a new entity definition with given parameters + * Sets the flag whether the entity has at max one vector per segment * - * @param entityName The name of the entity - * @param attributes A list of attributes - * @param oneVectorPerSegment Whether this entity will have one vector per segment at max + * @return The builder */ - private EntityDefinition(String entityName, List attributes, boolean oneVectorPerSegment) { - this.entityName = entityName; - this.attributes = attributes; - this.oneVectorPerSegment = oneVectorPerSegment; + public EntityDefinitionBuilder producesOneVectorPerSegment() { + this.oneVectorPerSegment = true; + return this; } /** - * Returns the name of the entity + * Adds an attribute to the entity * - * @return The name of the entity + * @param attr The attribute to add + * @return The builder */ - public String getEntityName() { - return entityName; + public EntityDefinitionBuilder withAttribute(AttributeDefinition attr) { + this.attrs.add(attr); + return this; } /** - * Returns an immutable list of attributes + * Adds a vector attribute with a given name and specified length * - * @return An immutable list of attributes + * @param featureName The name of the feature + * @param length The length of the vector + * @return The builder */ - public List getAttributes() { - return Collections.unmodifiableList(attributes); + public EntityDefinitionBuilder withAttribute(String featureName, int length) { + this.attrs.add(new AttributeDefinition(featureName, AttributeDefinition.AttributeType.VECTOR, length)); + return this; } /** - * Whether this entity will have one vector per segment at max + * Adds all given attributes * - * @return This entity will have one vector per segment at max + * @param attributes The attributes to add + * @return The builder */ - public boolean isOneVectorPerSegment() { - return oneVectorPerSegment; + public EntityDefinitionBuilder withAttributes(AttributeDefinition... attributes) { + this.attrs.addAll(Arrays.asList(attributes)); + return this; } - @Override - public String toString() { - return "EntityDefinition{" + - "entityName='" + entityName + '\'' + - ", attributes=" + attributes + - ", oneVectorPerSegment=" + oneVectorPerSegment + - '}'; + /** + * Adds many named vector attributes with the same length + * + * @param length The length of the vectors + * @param featureNames the names + * @return The builder + */ + public EntityDefinitionBuilder withAttributes(int length, String... featureNames) { + Arrays.stream(featureNames).forEach(s -> withAttribute(s, length)); + return this; } /** - * A builder for {@link EntityDefinition}s + * Adds a special 'id' attribute of type {@link AttributeDefinition.AttributeType#STRING} + * + * @return the builder */ - public static class EntityDefinitionBuilder { - private String name; - private List attrs = new ArrayList<>(); - private boolean oneVectorPerSegment; - - /** - * Creates a new builder for a named entity - * - * @param name The name of the entity - */ - public EntityDefinitionBuilder(String name) { - this.name = name; - } - - /** - * Creates a new builder from an entity defintiion, to modify said one - * @param def The defintiion template for this builder - * @return A new builder - */ - public static EntityDefinitionBuilder from(EntityDefinition def) { - EntityDefinitionBuilder builder = new EntityDefinitionBuilder(def.getEntityName()); - builder.attrs.addAll(def.getAttributes()); - builder.oneVectorPerSegment = def.oneVectorPerSegment; - return builder; - } - - public void reset(){ - name = ""; - attrs.clear(); - oneVectorPerSegment = false; - } - - /** - * Sets the name of the entity to build - * - * @param name The name of the entity - * @return The builder - */ - public EntityDefinitionBuilder withName(String name) { - this.name = name; - return this; - } - - /** - * Sets the flag whether the entity has at max one vector per segment - * - * @return The builder - */ - public EntityDefinitionBuilder producesOneVectorPerSegment() { - this.oneVectorPerSegment = true; - return this; - } - - /** - * Adds an attribute to the entity - * - * @param attr The attribute to add - * @return The builder - */ - public EntityDefinitionBuilder withAttribute(AttributeDefinition attr) { - this.attrs.add(attr); - return this; - } - - /** - * Adds a vector attribute with a given name and specified length - * - * @param featureName The name of the feature - * @param length The length of the vector - * @return The builder - */ - public EntityDefinitionBuilder withAttribute(String featureName, int length) { - this.attrs.add(new AttributeDefinition(featureName, AttributeDefinition.AttributeType.VECTOR, length)); - return this; - } - - /** - * Adds all given attributes - * - * @param attributes The attributes to add - * @return The builder - */ - public EntityDefinitionBuilder withAttributes(AttributeDefinition... attributes) { - this.attrs.addAll(Arrays.asList(attributes)); - return this; - } - - /** - * Adds many named vector attributes with the same length - * - * @param length The length of the vectors - * @param featureNames the names - * @return The builder - */ - public EntityDefinitionBuilder withAttributes(int length, String... featureNames) { - Arrays.stream(featureNames).forEach(s -> withAttribute(s, length)); - return this; - } - - /** - * Adds a special 'id' attribute of type {@link AttributeDefinition.AttributeType#STRING} - * - * @return the builder - */ - public EntityDefinitionBuilder withIdAttribute() { - // TODO add hash hint - this.attrs.add(0, new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING)); - return this; - } + public EntityDefinitionBuilder withIdAttribute() { + // TODO add hash hint + this.attrs.add(0, new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING)); + return this; + } - /** - * Builds the entity definition - * - * @return The entity definition - */ - public EntityDefinition build() { - // First sort so that id column is first - attrs.sort((o1, o2) -> { - if (o1.getName().equals("id")) { - return Integer.MIN_VALUE; - } else if (o2.getName().equals("id")) { - return Integer.MAX_VALUE; - } else { - return o1.getName().compareToIgnoreCase(o2.getName()); - } - }); - return new EntityDefinition(name, attrs, oneVectorPerSegment); + /** + * Builds the entity definition + * + * @return The entity definition + */ + public EntityDefinition build() { + // First sort so that id column is first + attrs.sort((o1, o2) -> { + if (o1.getName().equals("id")) { + return Integer.MIN_VALUE; + } else if (o2.getName().equals("id")) { + return Integer.MAX_VALUE; + } else { + return o1.getName().compareToIgnoreCase(o2.getName()); } + }); + return new EntityDefinition(name, attrs, oneVectorPerSegment); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java index a19ddcb7a..18e6646c6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java @@ -19,7 +19,9 @@ public boolean createMetadataEntity(String tableName) { } @Override - public boolean createSegmentMetadataEntity() { return false; } + public boolean createSegmentMetadataEntity() { + return false; + } @Override public boolean createSegmentMetadataEntity(String tableName) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java index 2fd462aa3..34787ed78 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/AvgImg.java @@ -1,63 +1,62 @@ package org.vitrivr.cineast.core.descriptor; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ReadableRGBContainer; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.util.DecodingError; import org.vitrivr.cineast.core.util.TimeHelper; -import java.util.List; - public class AvgImg { - private static final Logger LOGGER = LogManager.getLogger(); - - private AvgImg(){} - - public static MultiImage getAvg(List videoFrames){ - TimeHelper.tic(); - LOGGER.traceEntry(); - MultiImage first = videoFrames.get(0).getImage(); - int width = first.getWidth(), height = first.getHeight(); - double[] buffer = new double[width * height * 3]; - int[] colors; - try{ - for(VideoFrame videoFrame : videoFrames){ - colors = videoFrame.getImage().getColors(); - if((colors.length * 3) != buffer.length){ - throw new DecodingError(); - } - for(int i = 0; i < colors.length; ++i){ - int col = colors[i]; - buffer[3*i] += ReadableRGBContainer.getRed(col); - buffer[3*i + 1] += ReadableRGBContainer.getGreen(col); - buffer[3*i + 2] += ReadableRGBContainer.getBlue(col); - } - } - }catch(Exception e){ - throw new DecodingError(); - } - - - int size = videoFrames.size(); - - colors = new int[width * height]; - - for(int i = 0; i < colors.length; ++i){ - colors[i] = ReadableRGBContainer.toIntColor( - (int)Math.round(buffer[3*i] / size), - (int)Math.round(buffer[3*i + 1] / size), - (int)Math.round(buffer[3*i + 2] / size)); - } - - buffer = null; - - System.gc(); - LOGGER.debug("AvgImg.getAvg() done in {}", TimeHelper.toc()); - LOGGER.traceExit(); - return first.factory().newMultiImage(width, height, colors); - } - + private static final Logger LOGGER = LogManager.getLogger(); + + private AvgImg() { + } + + public static MultiImage getAvg(List videoFrames) { + TimeHelper.tic(); + LOGGER.traceEntry(); + MultiImage first = videoFrames.get(0).getImage(); + int width = first.getWidth(), height = first.getHeight(); + double[] buffer = new double[width * height * 3]; + int[] colors; + try { + for (VideoFrame videoFrame : videoFrames) { + colors = videoFrame.getImage().getColors(); + if ((colors.length * 3) != buffer.length) { + throw new DecodingError(); + } + for (int i = 0; i < colors.length; ++i) { + int col = colors[i]; + buffer[3 * i] += ReadableRGBContainer.getRed(col); + buffer[3 * i + 1] += ReadableRGBContainer.getGreen(col); + buffer[3 * i + 2] += ReadableRGBContainer.getBlue(col); + } + } + } catch (Exception e) { + throw new DecodingError(); + } + + int size = videoFrames.size(); + + colors = new int[width * height]; + + for (int i = 0; i < colors.length; ++i) { + colors[i] = ReadableRGBContainer.toIntColor( + (int) Math.round(buffer[3 * i] / size), + (int) Math.round(buffer[3 * i + 1] / size), + (int) Math.round(buffer[3 * i + 2] / size)); + } + + buffer = null; + + System.gc(); + LOGGER.debug("AvgImg.getAvg() done in {}", TimeHelper.toc()); + LOGGER.traceExit(); + return first.factory().newMultiImage(width, height, colors); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeImg.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeImg.java index 242aca742..fdeae6e1c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeImg.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeImg.java @@ -9,114 +9,116 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; - -import java.awt.*; +import java.awt.Color; +import java.awt.Graphics; import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; public class EdgeImg { - private EdgeImg() { - } - - private static final int CACHE_SIZE = 8; - - private static final Logger LOGGER = LogManager.getLogger(); - - private static final float THRESHOLD_LOW = 0.075f, THRESHOLD_HIGH = 0.3f; - - - public static MultiImage getEdgeImg(MultiImage img) { - LOGGER.traceEntry(); - - GrayU8 gray = ConvertBufferedImage.convertFrom(img.getBufferedImage(), (GrayU8) null); - if(!isSolid(gray)){ - getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); - } - - BufferedImage bout = VisualizeBinaryData.renderBinary(gray, false, null); - - return LOGGER.traceExit(img.factory().newMultiImage(bout)); - } - - public static boolean[] getEdgePixels(MultiImage img, boolean[] out) { - LOGGER.traceEntry(); - - if (out == null || out.length != img.getWidth() * img.getHeight()) { - out = new boolean[img.getWidth() * img.getHeight()]; - } - - GrayU8 gray = ConvertBufferedImage.convertFrom(img.getBufferedImage(), (GrayU8) null); - - if(!isSolid(gray)){ - getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); - - } - - for (int i = 0; i < gray.data.length; ++i) { - out[i] = (gray.data[i] != 0); - } - - LOGGER.traceExit(); - return out; - } - - public static List getEdgePixels(MultiImage img, List out) { - LOGGER.traceEntry(); - if (out == null) { - out = new ArrayList(img.getWidth() * img.getHeight()); - } else { - out.clear(); - } - - BufferedImage withBackground = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB); - Graphics g = withBackground.getGraphics(); - g.setColor(Color.white); - g.fillRect(0, 0, img.getWidth(), img.getHeight()); - g.drawImage(img.getBufferedImage(), 0, 0, null); - - GrayU8 gray = ConvertBufferedImage.convertFrom(withBackground, (GrayU8) null); - if(!isSolid(gray)){ - getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); - } - - for (int i = 0; i < gray.data.length; ++i) { - out.add(gray.data[i] != 0); - } - LOGGER.traceExit(); - return out; - } - - public static boolean isSolid(GrayU8 img){ - byte first = img.data[0]; - for(byte b : img.data){ - if(b != first){ - return false; - } - } - return true; - } - - - private static LoadingCache> cannies = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE) - .expireAfterAccess(10, TimeUnit.MINUTES).build(new CacheLoader>(){ - - @Override - public CannyEdge load(Thread arg0){ - return FactoryEdgeDetectors.canny(2, false, true, GrayU8.class, GrayS16.class); - }}); - private static synchronized CannyEdge getCanny(){ - Thread current = Thread.currentThread(); - try { - return cannies.get(current); - } catch (ExecutionException e) { - return null; //NEVER HAPPENS - } - } + private EdgeImg() { + } + + private static final int CACHE_SIZE = 8; + + private static final Logger LOGGER = LogManager.getLogger(); + + private static final float THRESHOLD_LOW = 0.075f, THRESHOLD_HIGH = 0.3f; + + + public static MultiImage getEdgeImg(MultiImage img) { + LOGGER.traceEntry(); + + GrayU8 gray = ConvertBufferedImage.convertFrom(img.getBufferedImage(), (GrayU8) null); + if (!isSolid(gray)) { + getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); + } + + BufferedImage bout = VisualizeBinaryData.renderBinary(gray, false, null); + + return LOGGER.traceExit(img.factory().newMultiImage(bout)); + } + + public static boolean[] getEdgePixels(MultiImage img, boolean[] out) { + LOGGER.traceEntry(); + + if (out == null || out.length != img.getWidth() * img.getHeight()) { + out = new boolean[img.getWidth() * img.getHeight()]; + } + + GrayU8 gray = ConvertBufferedImage.convertFrom(img.getBufferedImage(), (GrayU8) null); + + if (!isSolid(gray)) { + getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); + + } + + for (int i = 0; i < gray.data.length; ++i) { + out[i] = (gray.data[i] != 0); + } + + LOGGER.traceExit(); + return out; + } + + public static List getEdgePixels(MultiImage img, List out) { + LOGGER.traceEntry(); + if (out == null) { + out = new ArrayList(img.getWidth() * img.getHeight()); + } else { + out.clear(); + } + + BufferedImage withBackground = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB); + Graphics g = withBackground.getGraphics(); + g.setColor(Color.white); + g.fillRect(0, 0, img.getWidth(), img.getHeight()); + g.drawImage(img.getBufferedImage(), 0, 0, null); + + GrayU8 gray = ConvertBufferedImage.convertFrom(withBackground, (GrayU8) null); + if (!isSolid(gray)) { + getCanny().process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, gray); + } + + for (int i = 0; i < gray.data.length; ++i) { + out.add(gray.data[i] != 0); + } + LOGGER.traceExit(); + return out; + } + + public static boolean isSolid(GrayU8 img) { + byte first = img.data[0]; + for (byte b : img.data) { + if (b != first) { + return false; + } + } + return true; + } + + + private static LoadingCache> cannies = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE) + .expireAfterAccess(10, TimeUnit.MINUTES).build(new CacheLoader>() { + + @Override + public CannyEdge load(Thread arg0) { + return FactoryEdgeDetectors.canny(2, false, true, GrayU8.class, GrayS16.class); + } + }); + + private static synchronized CannyEdge getCanny() { + Thread current = Thread.currentThread(); + try { + return cannies.get(current); + } catch (ExecutionException e) { + return null; //NEVER HAPPENS + } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeList.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeList.java index 95f56e353..7111242e1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeList.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/EdgeList.java @@ -9,61 +9,61 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; - -import java.awt.*; +import java.awt.Color; +import java.awt.Graphics; import java.awt.image.BufferedImage; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; public class EdgeList { - private EdgeList() { - } + private EdgeList() { + } + + private static final int CACHE_SIZE = 8; - private static final int CACHE_SIZE = 8; + private static final Logger LOGGER = LogManager.getLogger(); + private static final float THRESHOLD_LOW = 0.1f, THRESHOLD_HIGH = 0.3f; - private static final Logger LOGGER = LogManager.getLogger(); - private static final float THRESHOLD_LOW = 0.1f, THRESHOLD_HIGH = 0.3f; + public static List getEdgeList(MultiImage img) { + LOGGER.traceEntry(); + BufferedImage withBackground = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB); + Graphics g = withBackground.getGraphics(); + g.setColor(Color.white); + g.fillRect(0, 0, img.getWidth(), img.getHeight()); + g.drawImage(img.getBufferedImage(), 0, 0, null); + GrayU8 gray = ConvertBufferedImage.convertFrom(withBackground, (GrayU8) null); + CannyEdge canny = getCanny(); + canny.process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, null); + List _return = canny.getContours(); + LOGGER.traceExit(); + return _return; + } - public static List getEdgeList(MultiImage img){ - LOGGER.traceEntry(); - BufferedImage withBackground = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB); - Graphics g = withBackground.getGraphics(); - g.setColor(Color.white); - g.fillRect(0, 0, img.getWidth(), img.getHeight()); - g.drawImage(img.getBufferedImage(), 0, 0, null); - GrayU8 gray = ConvertBufferedImage.convertFrom(withBackground, (GrayU8) null); - CannyEdge canny = getCanny(); - canny.process(gray, THRESHOLD_LOW, THRESHOLD_HIGH, null); - List _return = canny.getContours(); - LOGGER.traceExit(); - return _return; - } - - private static LoadingCache> cannies = CacheBuilder - .newBuilder() - .maximumSize(CACHE_SIZE) - .expireAfterAccess(10, TimeUnit.MINUTES) - .build(new CacheLoader>() { + private static LoadingCache> cannies = CacheBuilder + .newBuilder() + .maximumSize(CACHE_SIZE) + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(new CacheLoader>() { - @Override - public CannyEdge load(Thread arg0) { - return FactoryEdgeDetectors.canny(2, true, true, - GrayU8.class, GrayS16.class); - } - }); + @Override + public CannyEdge load(Thread arg0) { + return FactoryEdgeDetectors.canny(2, true, true, + GrayU8.class, GrayS16.class); + } + }); - private static synchronized CannyEdge getCanny() { - Thread current = Thread.currentThread(); - try { - return cannies.get(current); - } catch (ExecutionException e) { - return null; // NEVER HAPPENS - } - } + private static synchronized CannyEdge getCanny() { + Thread current = Thread.currentThread(); + try { + return cannies.get(current); + } catch (ExecutionException e) { + return null; // NEVER HAPPENS + } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/ImageDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/ImageDistance.java index 4b90dcd8b..622e40c65 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/ImageDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/ImageDistance.java @@ -5,40 +5,41 @@ public class ImageDistance { - private ImageDistance(){} - - public static double colorDistance(int col1, int col2){ - int r1 = ReadableRGBContainer.getRed(col1), r2 = ReadableRGBContainer.getRed(col2), - g1 = ReadableRGBContainer.getGreen(col1), g2 = ReadableRGBContainer.getGreen(col2), - b1 = ReadableRGBContainer.getBlue(col1), b2 = ReadableRGBContainer.getBlue(col2); - - return Math.sqrt((r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2) + (b1 - b2) * (b1 - b2)); - } - - public static double colorDistance(int[] img1, int[] img2){ - if(img1.length != img2.length){ - return Double.POSITIVE_INFINITY; - } - double distance = 0; - for(int i = 0; i < img1.length; ++i){ - distance += colorDistance(img1[i], img2[i]); - } - distance /= img1.length; - return distance; - } - - public static double colorDistanceFull(MultiImage img1, MultiImage img2){ - if(img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()){ - return Double.POSITIVE_INFINITY; - } - return colorDistance(img1.getColors(), img2.getColors()); - } - - public static double colorDistance(MultiImage img1, MultiImage img2){ - if(img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()){ - return Double.POSITIVE_INFINITY; - } - return colorDistance(img1.getThumbnailColors(), img2.getThumbnailColors()); - } - + private ImageDistance() { + } + + public static double colorDistance(int col1, int col2) { + int r1 = ReadableRGBContainer.getRed(col1), r2 = ReadableRGBContainer.getRed(col2), + g1 = ReadableRGBContainer.getGreen(col1), g2 = ReadableRGBContainer.getGreen(col2), + b1 = ReadableRGBContainer.getBlue(col1), b2 = ReadableRGBContainer.getBlue(col2); + + return Math.sqrt((r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2) + (b1 - b2) * (b1 - b2)); + } + + public static double colorDistance(int[] img1, int[] img2) { + if (img1.length != img2.length) { + return Double.POSITIVE_INFINITY; + } + double distance = 0; + for (int i = 0; i < img1.length; ++i) { + distance += colorDistance(img1[i], img2[i]); + } + distance /= img1.length; + return distance; + } + + public static double colorDistanceFull(MultiImage img1, MultiImage img2) { + if (img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()) { + return Double.POSITIVE_INFINITY; + } + return colorDistance(img1.getColors(), img2.getColors()); + } + + public static double colorDistance(MultiImage img1, MultiImage img2) { + if (img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()) { + return Double.POSITIVE_INFINITY; + } + return colorDistance(img1.getThumbnailColors(), img2.getThumbnailColors()); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java index 2c7a95027..6365d4df0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MedianImg.java @@ -1,124 +1,123 @@ package org.vitrivr.cineast.core.descriptor; +import java.util.Arrays; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.RGBContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.util.DecodingError; -import java.util.Arrays; -import java.util.List; /** * WARNING: EXTREMELY SLOW - * - * */ public class MedianImg { - private static final Logger LOGGER = LogManager.getLogger(); - - private MedianImg(){} - - public static MultiImage getMedian(List videoFrames){ - - LOGGER.traceEntry(); - - MultiImage first = videoFrames.get(0).getImage(); - int width = first.getWidth(), height = first.getHeight(); - - short[] buffer = new short[width * height * 128]; - - int[] result = new int[width * height]; - - //red pass - try{ - for(VideoFrame f : videoFrames){ - int[] colors = f.getImage().getColors(); - for(int i = 0; i < colors.length; ++i){ - buffer[128 * i + (RGBContainer.getRed(colors[i])) / 2]++; - } - colors = null; - } - }catch(Exception e){ - throw new DecodingError(); - } - short[] hist = new short[128]; - - for(int i = 0; i < result.length; ++i){ - System.arraycopy(buffer, i * 128, hist, 0, 128); - int r = medianFromHistogram(hist) * 2; - result[i] = r; - } - - //green pass - - Arrays.fill(buffer, (short)0); - try{ - for(VideoFrame f : videoFrames){ - int[] colors = f.getImage().getColors(); - for(int i = 0; i < colors.length; ++i){ - buffer[128 * i + (RGBContainer.getGreen(colors[i])) / 2]++; - } - colors = null; - } - }catch(Exception e){ - throw new DecodingError(); - } - - for(int i = 0; i < result.length; ++i){ - System.arraycopy(buffer, i * 128, hist, 0, 128); - int r = result[i]; - int g = medianFromHistogram(hist) * 2; - result[i] = ReadableRGBContainer.toIntColor(r, g, 0); - } - - //blue pass - Arrays.fill(buffer, (short)0); - try{ - for(VideoFrame f : videoFrames){ - int[] colors = f.getImage().getColors(); - for(int i = 0; i < colors.length; ++i){ - buffer[128 * i + (RGBContainer.getBlue(colors[i])) / 2]++; - } - colors = null; - } - }catch(Exception e){ - throw new DecodingError(); - } - - for(int i = 0; i < result.length; ++i){ - System.arraycopy(buffer, i * 128, hist, 0, 128); - int r = ReadableRGBContainer.getRed(result[i]); - int g = ReadableRGBContainer.getGreen(result[i]); - int b = medianFromHistogram(hist) * 2; - result[i] = ReadableRGBContainer.toIntColor(r, g, b); - } - - buffer = null; - - System.gc(); - LOGGER.traceExit(); - return first.factory().newMultiImage(width, height, result); - } - - private static int medianFromHistogram(short[] hist){ - int pos_l = 0, pos_r = hist.length - 1; - int sum_l = uShortToInt(hist[pos_l]), sum_r = uShortToInt(hist[pos_r]); - - while(pos_l < pos_r){ - if(sum_l < sum_r){ - sum_l += uShortToInt(hist[++pos_l]); - }else{ - sum_r += uShortToInt(hist[--pos_r]); - } - } - return pos_l; - } - - private static int uShortToInt(short s){ - return (s % 0xFFFF); - } - + private static final Logger LOGGER = LogManager.getLogger(); + + private MedianImg() { + } + + public static MultiImage getMedian(List videoFrames) { + + LOGGER.traceEntry(); + + MultiImage first = videoFrames.get(0).getImage(); + int width = first.getWidth(), height = first.getHeight(); + + short[] buffer = new short[width * height * 128]; + + int[] result = new int[width * height]; + + //red pass + try { + for (VideoFrame f : videoFrames) { + int[] colors = f.getImage().getColors(); + for (int i = 0; i < colors.length; ++i) { + buffer[128 * i + (RGBContainer.getRed(colors[i])) / 2]++; + } + colors = null; + } + } catch (Exception e) { + throw new DecodingError(); + } + short[] hist = new short[128]; + + for (int i = 0; i < result.length; ++i) { + System.arraycopy(buffer, i * 128, hist, 0, 128); + int r = medianFromHistogram(hist) * 2; + result[i] = r; + } + + //green pass + + Arrays.fill(buffer, (short) 0); + try { + for (VideoFrame f : videoFrames) { + int[] colors = f.getImage().getColors(); + for (int i = 0; i < colors.length; ++i) { + buffer[128 * i + (RGBContainer.getGreen(colors[i])) / 2]++; + } + colors = null; + } + } catch (Exception e) { + throw new DecodingError(); + } + + for (int i = 0; i < result.length; ++i) { + System.arraycopy(buffer, i * 128, hist, 0, 128); + int r = result[i]; + int g = medianFromHistogram(hist) * 2; + result[i] = ReadableRGBContainer.toIntColor(r, g, 0); + } + + //blue pass + Arrays.fill(buffer, (short) 0); + try { + for (VideoFrame f : videoFrames) { + int[] colors = f.getImage().getColors(); + for (int i = 0; i < colors.length; ++i) { + buffer[128 * i + (RGBContainer.getBlue(colors[i])) / 2]++; + } + colors = null; + } + } catch (Exception e) { + throw new DecodingError(); + } + + for (int i = 0; i < result.length; ++i) { + System.arraycopy(buffer, i * 128, hist, 0, 128); + int r = ReadableRGBContainer.getRed(result[i]); + int g = ReadableRGBContainer.getGreen(result[i]); + int b = medianFromHistogram(hist) * 2; + result[i] = ReadableRGBContainer.toIntColor(r, g, b); + } + + buffer = null; + + System.gc(); + LOGGER.traceExit(); + return first.factory().newMultiImage(width, height, result); + } + + private static int medianFromHistogram(short[] hist) { + int pos_l = 0, pos_r = hist.length - 1; + int sum_l = uShortToInt(hist[pos_l]), sum_r = uShortToInt(hist[pos_r]); + + while (pos_l < pos_r) { + if (sum_l < sum_r) { + sum_l += uShortToInt(hist[++pos_l]); + } else { + sum_r += uShortToInt(hist[--pos_r]); + } + } + return pos_l; + } + + private static int uShortToInt(short s) { + return (s % 0xFFFF); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MostRepresentative.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MostRepresentative.java index 804a94f75..5a4695734 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MostRepresentative.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/MostRepresentative.java @@ -2,29 +2,30 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.VideoSegment; public class MostRepresentative { - private static final Logger LOGGER = LogManager.getLogger(); - - private MostRepresentative(){} - - public static VideoFrame getMostRepresentative(VideoSegment videoSegment){ - LOGGER.traceEntry(); - MultiImage reference = videoSegment.getAvgImg(); - VideoFrame _return = null; - double minDist = Double.POSITIVE_INFINITY; - for(VideoFrame f : videoSegment.getVideoFrames()){ - double dist = ImageDistance.colorDistance(reference, f.getImage()); - if(dist < minDist){ - minDist = dist; - _return = f; - } - } - return LOGGER.traceExit(_return); - } - + private static final Logger LOGGER = LogManager.getLogger(); + + private MostRepresentative() { + } + + public static VideoFrame getMostRepresentative(VideoSegment videoSegment) { + LOGGER.traceEntry(); + MultiImage reference = videoSegment.getAvgImg(); + VideoFrame _return = null; + double minDist = Double.POSITIVE_INFINITY; + for (VideoFrame f : videoSegment.getVideoFrames()) { + double dist = ImageDistance.colorDistance(reference, f.getImage()); + if (dist < minDist) { + minDist = dist; + _return = f; + } + } + return LOGGER.traceExit(_return); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/PathList.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/PathList.java index 467061b92..d053b5622 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/PathList.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/descriptor/PathList.java @@ -23,229 +23,228 @@ import georegression.struct.point.Point2D_F32; import georegression.struct.point.Point2D_F64; import georegression.transform.homography.HomographyPointOps_F64; -import org.ddogleg.fitting.modelset.ModelMatcher; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.frames.VideoFrame; - import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; +import org.ddogleg.fitting.modelset.ModelMatcher; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.frames.VideoFrame; public class PathList { - private PathList() { - } + private PathList() { + } - public static int samplingInterval = 15; - public static int frameInterval = 2; + public static int samplingInterval = 15; + public static int frameInterval = 2; - public static double backwardTrackingDistanceThreshold = 5.0; - public static double successTrackingRatioThreshold = 0.50; - public static double ransacInlierRatioThreshold = 0.65; - public static double successFrameSRatioThreshold = 0.70; + public static double backwardTrackingDistanceThreshold = 5.0; + public static double successTrackingRatioThreshold = 0.50; + public static double ransacInlierRatioThreshold = 0.65; + public static double successFrameSRatioThreshold = 0.70; - public static void showBineryImage(GrayU8 image) { - PixelMath.multiply(image, 255, image); - BufferedImage out = ConvertBufferedImage.convertTo(image, null); - ShowImages.showWindow(out, "Output"); - } + public static void showBineryImage(GrayU8 image) { + PixelMath.multiply(image, 255, image); + BufferedImage out = ConvertBufferedImage.convertTo(image, null); + ShowImages.showWindow(out, "Output"); + } - public static void separateFgBgPaths(List videoFrames, - LinkedList>> allPaths, - List>> foregroundPaths, - List>> backgroundPaths) { + public static void separateFgBgPaths(List videoFrames, + LinkedList>> allPaths, + List>> foregroundPaths, + List>> backgroundPaths) { - ModelMatcher robustF = FactoryMultiViewRobust.homographyRansac(null, new ConfigRansac(200, 3.0f)); - if (allPaths == null || videoFrames == null || videoFrames.isEmpty()) { - return; - } - - int width = videoFrames.get(0).getImage().getWidth(); - int height = videoFrames.get(0).getImage().getHeight(); - int maxTracksNumber = (width * height) / (samplingInterval * samplingInterval); - int failedFrameCount = 0; - - for (Pair> pair : allPaths) { - List inliers = new ArrayList(); - List outliers = new ArrayList(); - - int frameIdx = pair.first; - ArrayList matches = pair.second; - - if (matches.size() < maxTracksNumber * successTrackingRatioThreshold) { - failedFrameCount += 1; - continue; - } - - Homography2D_F64 curToPrev = new Homography2D_F64(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0); - if (robustF.process(matches) && robustF.getMatchSet().size() > matches.size() * ransacInlierRatioThreshold) { - curToPrev = robustF.getModelParameters().invert(null); - inliers.addAll(robustF.getMatchSet()); - for (int i = 0, j = 0; i < matches.size(); ++i) { - if (i == robustF.getInputIndex(j)) { - if (j < inliers.size() - 1) { - ++j; - } - } else { - outliers.add(matches.get(i)); - } - } - } else { - curToPrev = new Homography2D_F64(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0); - failedFrameCount += 1; - } - - for (AssociatedPair p : inliers) { - LinkedList path = new LinkedList(); - path.add(new Point2D_F32((float) p.p1.x / width, (float) p.p1.y / height)); - path.add(new Point2D_F32((float) p.p2.x / width, (float) p.p2.y / height)); - backgroundPaths.add(new Pair>(frameIdx, path)); - } + ModelMatcher robustF = FactoryMultiViewRobust.homographyRansac(null, new ConfigRansac(200, 3.0f)); + if (allPaths == null || videoFrames == null || videoFrames.isEmpty()) { + return; + } - for (AssociatedPair p : outliers) { - p.p2 = HomographyPointOps_F64.transform(curToPrev, p.p2, p.p2); - LinkedList path = new LinkedList(); - path.add(new Point2D_F32((float) p.p1.x / width, (float) p.p1.y / height)); - path.add(new Point2D_F32((float) p.p2.x / width, (float) p.p2.y / height)); - foregroundPaths.add(new Pair>(frameIdx, path)); + int width = videoFrames.get(0).getImage().getWidth(); + int height = videoFrames.get(0).getImage().getHeight(); + int maxTracksNumber = (width * height) / (samplingInterval * samplingInterval); + int failedFrameCount = 0; + + for (Pair> pair : allPaths) { + List inliers = new ArrayList(); + List outliers = new ArrayList(); + + int frameIdx = pair.first; + ArrayList matches = pair.second; + + if (matches.size() < maxTracksNumber * successTrackingRatioThreshold) { + failedFrameCount += 1; + continue; + } + + Homography2D_F64 curToPrev = new Homography2D_F64(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0); + if (robustF.process(matches) && robustF.getMatchSet().size() > matches.size() * ransacInlierRatioThreshold) { + curToPrev = robustF.getModelParameters().invert(null); + inliers.addAll(robustF.getMatchSet()); + for (int i = 0, j = 0; i < matches.size(); ++i) { + if (i == robustF.getInputIndex(j)) { + if (j < inliers.size() - 1) { + ++j; } + } else { + outliers.add(matches.get(i)); + } } - - int frameNum = allPaths.size(); - if (frameNum - failedFrameCount < frameNum * successFrameSRatioThreshold) { - foregroundPaths.clear(); - backgroundPaths.clear(); - } - - return; + } else { + curToPrev = new Homography2D_F64(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0); + failedFrameCount += 1; + } + + for (AssociatedPair p : inliers) { + LinkedList path = new LinkedList(); + path.add(new Point2D_F32((float) p.p1.x / width, (float) p.p1.y / height)); + path.add(new Point2D_F32((float) p.p2.x / width, (float) p.p2.y / height)); + backgroundPaths.add(new Pair>(frameIdx, path)); + } + + for (AssociatedPair p : outliers) { + p.p2 = HomographyPointOps_F64.transform(curToPrev, p.p2, p.p2); + LinkedList path = new LinkedList(); + path.add(new Point2D_F32((float) p.p1.x / width, (float) p.p1.y / height)); + path.add(new Point2D_F32((float) p.p2.x / width, (float) p.p2.y / height)); + foregroundPaths.add(new Pair>(frameIdx, path)); + } } - public static LinkedList>> getDensePaths(List videoFrames) { - if (videoFrames.size() < 2) { - return null; - } - - PkltConfig configKlt = new PkltConfig(3, new int[]{1, 2, 4}); - configKlt.config.maxPerPixelError = 45; - ImageGradient gradient = FactoryDerivative.sobel(GrayU8.class, GrayS16.class); - PyramidDiscrete pyramidForeward = FactoryPyramid.discreteGaussian(configKlt.pyramidScaling, -1, 2, true, ImageType.single(GrayU8.class)); - PyramidDiscrete pyramidBackward = FactoryPyramid.discreteGaussian(configKlt.pyramidScaling, -1, 2, true, ImageType.single(GrayU8.class)); - PyramidKltTracker trackerForeward = FactoryTrackerAlg.kltPyramid(configKlt.config, GrayU8.class, null); - PyramidKltTracker trackerBackward = FactoryTrackerAlg.kltPyramid(configKlt.config, GrayU8.class, null); - - GrayS16[] derivX = null; - GrayS16[] derivY = null; - - LinkedList tracks = new LinkedList(); - LinkedList>> paths = new LinkedList>>(); - - GrayU8 gray = null; - int frameIdx = 0; - int cnt = 0; - for (VideoFrame videoFrame : videoFrames) { - ++frameIdx; - - if (cnt >= frameInterval) { - cnt = 0; - continue; - } - cnt += 1; - - gray = ConvertBufferedImage.convertFrom(videoFrame.getImage().getBufferedImage(), gray); - ArrayList tracksPairs = new ArrayList(); + int frameNum = allPaths.size(); + if (frameNum - failedFrameCount < frameNum * successFrameSRatioThreshold) { + foregroundPaths.clear(); + backgroundPaths.clear(); + } - if (frameIdx == 0) { - tracks = denseSampling(gray, derivX, derivY, samplingInterval, configKlt, gradient, pyramidBackward, trackerBackward); - } else { - tracking(gray, derivX, derivY, tracks, tracksPairs, gradient, pyramidForeward, pyramidBackward, trackerForeward, trackerBackward); - tracks = denseSampling(gray, derivX, derivY, samplingInterval, configKlt, gradient, pyramidBackward, trackerBackward); - } + return; + } - paths.add(new Pair>(frameIdx, tracksPairs)); - } - return paths; + public static LinkedList>> getDensePaths(List videoFrames) { + if (videoFrames.size() < 2) { + return null; } - public static LinkedList denseSampling(GrayU8 image, GrayS16[] derivX, GrayS16[] derivY, - int samplingInterval, - PkltConfig configKlt, - ImageGradient gradient, - PyramidDiscrete pyramid, - PyramidKltTracker tracker) { - LinkedList tracks = new LinkedList(); - - pyramid.process(image); - derivX = declareOutput(pyramid, derivX); - derivY = declareOutput(pyramid, derivY); - PyramidOps.gradient(pyramid, gradient, derivX, derivY); - tracker.setImage(pyramid, derivX, derivY); - - for (int y = 0; y < image.height; y += samplingInterval) { - for (int x = 0; x < image.width; x += samplingInterval) { - PyramidKltFeature t = new PyramidKltFeature(configKlt.pyramidScaling.length, configKlt.templateRadius); - t.setPosition(x, y); - tracker.setDescription(t); - tracks.add(t); - } - } - return tracks; + PkltConfig configKlt = new PkltConfig(3, new int[]{1, 2, 4}); + configKlt.config.maxPerPixelError = 45; + ImageGradient gradient = FactoryDerivative.sobel(GrayU8.class, GrayS16.class); + PyramidDiscrete pyramidForeward = FactoryPyramid.discreteGaussian(configKlt.pyramidScaling, -1, 2, true, ImageType.single(GrayU8.class)); + PyramidDiscrete pyramidBackward = FactoryPyramid.discreteGaussian(configKlt.pyramidScaling, -1, 2, true, ImageType.single(GrayU8.class)); + PyramidKltTracker trackerForeward = FactoryTrackerAlg.kltPyramid(configKlt.config, GrayU8.class, null); + PyramidKltTracker trackerBackward = FactoryTrackerAlg.kltPyramid(configKlt.config, GrayU8.class, null); + + GrayS16[] derivX = null; + GrayS16[] derivY = null; + + LinkedList tracks = new LinkedList(); + LinkedList>> paths = new LinkedList>>(); + + GrayU8 gray = null; + int frameIdx = 0; + int cnt = 0; + for (VideoFrame videoFrame : videoFrames) { + ++frameIdx; + + if (cnt >= frameInterval) { + cnt = 0; + continue; + } + cnt += 1; + + gray = ConvertBufferedImage.convertFrom(videoFrame.getImage().getBufferedImage(), gray); + ArrayList tracksPairs = new ArrayList(); + + if (frameIdx == 0) { + tracks = denseSampling(gray, derivX, derivY, samplingInterval, configKlt, gradient, pyramidBackward, trackerBackward); + } else { + tracking(gray, derivX, derivY, tracks, tracksPairs, gradient, pyramidForeward, pyramidBackward, trackerForeward, trackerBackward); + tracks = denseSampling(gray, derivX, derivY, samplingInterval, configKlt, gradient, pyramidBackward, trackerBackward); + } + + paths.add(new Pair>(frameIdx, tracksPairs)); } - - public static LinkedList tracking(GrayU8 image, GrayS16[] derivX, GrayS16[] derivY, - LinkedList tracks, - ArrayList tracksPairs, - ImageGradient gradient, - PyramidDiscrete pyramidForeward, - PyramidDiscrete pyramidBackward, - PyramidKltTracker trackerForeward, - PyramidKltTracker trackerBackward - ) { - pyramidForeward.process(image); - derivX = declareOutput(pyramidForeward, derivX); - derivY = declareOutput(pyramidForeward, derivY); - PyramidOps.gradient(pyramidForeward, gradient, derivX, derivY); - trackerForeward.setImage(pyramidForeward, derivX, derivY); - - ListIterator listIterator = tracks.listIterator(); - while (listIterator.hasNext()) { - PyramidKltFeature track = listIterator.next(); - Point2D_F64 pointPrev = new Point2D_F64(track.x, track.y); - KltTrackFault ret = trackerForeward.track(track); - boolean success = false; - if (ret == KltTrackFault.SUCCESS && image.isInBounds((int) track.x, (int) track.y) && trackerForeward.setDescription(track)) { - Point2D_F64 pointCur = new Point2D_F64(track.x, track.y); - ret = trackerBackward.track(track); - if (ret == KltTrackFault.SUCCESS && image.isInBounds((int) track.x, (int) track.y)) { - Point2D_F64 pointCurBack = new Point2D_F64(track.x, track.y); - if (normalizedDistance(pointPrev, pointCurBack) < backwardTrackingDistanceThreshold) { - tracksPairs.add(new AssociatedPair(pointPrev, pointCur)); - success = true; - } - } - } - if (!success) { - listIterator.remove(); - } - } - return tracks; + return paths; + } + + public static LinkedList denseSampling(GrayU8 image, GrayS16[] derivX, GrayS16[] derivY, + int samplingInterval, + PkltConfig configKlt, + ImageGradient gradient, + PyramidDiscrete pyramid, + PyramidKltTracker tracker) { + LinkedList tracks = new LinkedList(); + + pyramid.process(image); + derivX = declareOutput(pyramid, derivX); + derivY = declareOutput(pyramid, derivY); + PyramidOps.gradient(pyramid, gradient, derivX, derivY); + tracker.setImage(pyramid, derivX, derivY); + + for (int y = 0; y < image.height; y += samplingInterval) { + for (int x = 0; x < image.width; x += samplingInterval) { + PyramidKltFeature t = new PyramidKltFeature(configKlt.pyramidScaling.length, configKlt.templateRadius); + t.setPosition(x, y); + tracker.setDescription(t); + tracks.add(t); + } } - - public static GrayS16[] declareOutput(PyramidDiscrete pyramid, GrayS16[] deriv) { - if (deriv == null) { - deriv = PyramidOps.declareOutput(pyramid, GrayS16.class); - } else if (deriv[0].width != pyramid.getLayer(0).width || - deriv[0].height != pyramid.getLayer(0).height) { - PyramidOps.reshapeOutput(pyramid, deriv); + return tracks; + } + + public static LinkedList tracking(GrayU8 image, GrayS16[] derivX, GrayS16[] derivY, + LinkedList tracks, + ArrayList tracksPairs, + ImageGradient gradient, + PyramidDiscrete pyramidForeward, + PyramidDiscrete pyramidBackward, + PyramidKltTracker trackerForeward, + PyramidKltTracker trackerBackward + ) { + pyramidForeward.process(image); + derivX = declareOutput(pyramidForeward, derivX); + derivY = declareOutput(pyramidForeward, derivY); + PyramidOps.gradient(pyramidForeward, gradient, derivX, derivY); + trackerForeward.setImage(pyramidForeward, derivX, derivY); + + ListIterator listIterator = tracks.listIterator(); + while (listIterator.hasNext()) { + PyramidKltFeature track = listIterator.next(); + Point2D_F64 pointPrev = new Point2D_F64(track.x, track.y); + KltTrackFault ret = trackerForeward.track(track); + boolean success = false; + if (ret == KltTrackFault.SUCCESS && image.isInBounds((int) track.x, (int) track.y) && trackerForeward.setDescription(track)) { + Point2D_F64 pointCur = new Point2D_F64(track.x, track.y); + ret = trackerBackward.track(track); + if (ret == KltTrackFault.SUCCESS && image.isInBounds((int) track.x, (int) track.y)) { + Point2D_F64 pointCurBack = new Point2D_F64(track.x, track.y); + if (normalizedDistance(pointPrev, pointCurBack) < backwardTrackingDistanceThreshold) { + tracksPairs.add(new AssociatedPair(pointPrev, pointCur)); + success = true; + } } - return deriv; + } + if (!success) { + listIterator.remove(); + } } - - public static double normalizedDistance(Point2D_F64 pointA, Point2D_F64 pointB) { - double dx = (pointA.x - pointB.x); - double dy = (pointA.y - pointB.y); - return Math.sqrt(Math.pow((dx), 2) + Math.pow((dy), 2)); + return tracks; + } + + public static GrayS16[] declareOutput(PyramidDiscrete pyramid, GrayS16[] deriv) { + if (deriv == null) { + deriv = PyramidOps.declareOutput(pyramid, GrayS16.class); + } else if (deriv[0].width != pyramid.getLayer(0).width || + deriv[0].height != pyramid.getLayer(0).height) { + PyramidOps.reshapeOutput(pyramid, deriv); } + return deriv; + } + + public static double normalizedDistance(Point2D_F64 pointA, Point2D_F64 pointB) { + double dx = (pointA.x - pointB.x); + double dy = (pointA.y - pointB.y); + return Math.sqrt(Math.pow((dx), 2) + Math.pow((dy), 2)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java index 9548744b6..2effd4238 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java @@ -16,7 +16,6 @@ /** * Provides a configuration context for an extraction run. - * */ public interface ExtractionContextProvider { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/AudioDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/AudioDecoder.java index 7cf202126..25191028a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/AudioDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/AudioDecoder.java @@ -6,7 +6,7 @@ public interface AudioDecoder extends Decoder { - void seekToFrame(int frameNumber); + void seekToFrame(int frameNumber); - int getFrameNumber(); + int getFrameNumber(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/FFMpegAudioDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/FFMpegAudioDecoder.java index 0def7f3f9..60c466e7c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/FFMpegAudioDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/FFMpegAudioDecoder.java @@ -1,439 +1,447 @@ package org.vitrivr.cineast.core.extraction.decode.audio; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.avcodec; import org.bytedeco.javacpp.avcodec.AVCodec; import org.bytedeco.javacpp.avcodec.AVCodecContext; import org.bytedeco.javacpp.avcodec.AVPacket; +import org.bytedeco.javacpp.avformat; import org.bytedeco.javacpp.avformat.AVFormatContext; +import org.bytedeco.javacpp.avutil; import org.bytedeco.javacpp.avutil.AVDictionary; import org.bytedeco.javacpp.avutil.AVFrame; import org.bytedeco.javacpp.avutil.AVRational; +import org.bytedeco.javacpp.swresample; import org.bytedeco.javacpp.swresample.SwrContext; -import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.util.LogHelper; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayDeque; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - /** * A {@link Decoder} implementation that decodes audio using the ffmpeg library + the corresponding Java bindings. - * */ public class FFMpegAudioDecoder implements AudioDecoder { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Lists the mime-types supported by the FFMpegAudioDecoder. - * - * TODO: List may not be complete yet. */ - private static final Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("multimedia/mp4"); /* They share the same suffix with audio (.mp4). */ - tmp.add("audio/mp4"); - tmp.add("audio/aac"); - tmp.add("audio/mpeg"); - tmp.add("audio/ogg"); - tmp.add("audio/wav"); - tmp.add("audio/flac"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - - /** Property name and default value for channel settings. */ - private static final String CONFIG_CHANNELS_PROPERTY = "channels"; - private static final int CONFIG_CHANNELS_DEFAULT = 2; - - /** Property name and default value for target samplerate settings. */ - private static final String CONFIG_SAMPLERATE_PROPERTY = "samplerate"; - private static final int CONFIG_SAMPLERATE_DEFAULT = 44100; - - /** Default value for output sample format (16bit signed PCM). */ - private static final int TARGET_FORMAT = avutil.AV_SAMPLE_FMT_S16; - private static final int BYTES_PER_SAMPLE = avutil.av_get_bytes_per_sample(TARGET_FORMAT); - - /** Internal data structure used to hold decoded AudioFrames. */ - private ArrayDeque frameQueue = new ArrayDeque<>(); - - private AVFormatContext pFormatCtx = null; - private AVCodecContext pCodecCtx = null; - private int audioStream = -1; - - private AVPacket packet = null; - - private AVFrame decodedFrame = null; - private AVFrame resampledFrame = null; - - private IntPointer out_linesize = new IntPointer(); - - private SwrContext swr_ctx = null; - - private AtomicBoolean complete = new AtomicBoolean(false); - - private AudioDescriptor descriptor = null; - - /** - * Reads the next packet from the stream containing 1:n frames. If queue is set to true, - * the decoded frames are enqueued. - * - * @param queue If true, decoded frames are enqueued. Otherwise, they are discarded. - * @return True if frame was read, false otherwise. - */ - private boolean readFrame(boolean queue) { - boolean readFrame = false; - - /* Outer loop: Read packet (frame) from stream. */ - do { - int read_results = avformat.av_read_frame(this.pFormatCtx, this.packet); - if (read_results < 0 && !(read_results == avutil.AVERROR_EOF)) { - LOGGER.error("Error occurred while reading packet. FFMPEG av_read_frame() returned code {}.", read_results); - avcodec.av_packet_unref(this.packet); - break; - } + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Lists the mime-types supported by the FFMpegAudioDecoder. + *

    + * TODO: List may not be complete yet. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("multimedia/mp4"); /* They share the same suffix with audio (.mp4). */ + tmp.add("audio/mp4"); + tmp.add("audio/aac"); + tmp.add("audio/mpeg"); + tmp.add("audio/ogg"); + tmp.add("audio/wav"); + tmp.add("audio/flac"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Property name and default value for channel settings. + */ + private static final String CONFIG_CHANNELS_PROPERTY = "channels"; + private static final int CONFIG_CHANNELS_DEFAULT = 2; + + /** + * Property name and default value for target samplerate settings. + */ + private static final String CONFIG_SAMPLERATE_PROPERTY = "samplerate"; + private static final int CONFIG_SAMPLERATE_DEFAULT = 44100; + + /** + * Default value for output sample format (16bit signed PCM). + */ + private static final int TARGET_FORMAT = avutil.AV_SAMPLE_FMT_S16; + private static final int BYTES_PER_SAMPLE = avutil.av_get_bytes_per_sample(TARGET_FORMAT); + + /** + * Internal data structure used to hold decoded AudioFrames. + */ + private ArrayDeque frameQueue = new ArrayDeque<>(); + + private AVFormatContext pFormatCtx = null; + private AVCodecContext pCodecCtx = null; + private int audioStream = -1; + + private AVPacket packet = null; + + private AVFrame decodedFrame = null; + private AVFrame resampledFrame = null; + + private IntPointer out_linesize = new IntPointer(); + + private SwrContext swr_ctx = null; + + private AtomicBoolean complete = new AtomicBoolean(false); + + private AudioDescriptor descriptor = null; + + /** + * Reads the next packet from the stream containing 1:n frames. If queue is set to true, the decoded frames are enqueued. + * + * @param queue If true, decoded frames are enqueued. Otherwise, they are discarded. + * @return True if frame was read, false otherwise. + */ + private boolean readFrame(boolean queue) { + boolean readFrame = false; + + /* Outer loop: Read packet (frame) from stream. */ + do { + int read_results = avformat.av_read_frame(this.pFormatCtx, this.packet); + if (read_results < 0 && !(read_results == avutil.AVERROR_EOF)) { + LOGGER.error("Error occurred while reading packet. FFMPEG av_read_frame() returned code {}.", read_results); + avcodec.av_packet_unref(this.packet); + break; + } + + if (this.packet.stream_index() == this.audioStream) { + /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ + int decode_results = avcodec.avcodec_send_packet(this.pCodecCtx, read_results == avutil.AVERROR_EOF ? null : this.packet); + if (decode_results < 0) { + LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); + avcodec.av_packet_unref(this.packet); + break; + } - if (this.packet.stream_index() == this.audioStream) { - /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ - int decode_results = avcodec.avcodec_send_packet(this.pCodecCtx, read_results == avutil.AVERROR_EOF ? null : this.packet); - if (decode_results < 0) { - LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); - avcodec.av_packet_unref(this.packet); - break; - } - - /* Because a packet can theoretically contain more than one frame; repeat decoding until no samples are - * remaining in the packet. - */ - while (avcodec.avcodec_receive_frame(this.pCodecCtx, this.decodedFrame) == 0) { - /* If queue is true; enqueue frame. */ - if (queue) { - if (this.swr_ctx != null) { - this.readResampled(this.decodedFrame.nb_samples()); - } else { - this.readOriginal(this.decodedFrame.nb_samples()); - } - } - readFrame = true; - } + /* Because a packet can theoretically contain more than one frame; repeat decoding until no samples are + * remaining in the packet. + */ + while (avcodec.avcodec_receive_frame(this.pCodecCtx, this.decodedFrame) == 0) { + /* If queue is true; enqueue frame. */ + if (queue) { + if (this.swr_ctx != null) { + this.readResampled(this.decodedFrame.nb_samples()); + } else { + this.readOriginal(this.decodedFrame.nb_samples()); } - - /* Free the packet that was allocated by av_read_frame. */ - avcodec.av_packet_unref(this.packet); - } while(!readFrame); - - return readFrame; + } + readFrame = true; + } + } + + /* Free the packet that was allocated by av_read_frame. */ + avcodec.av_packet_unref(this.packet); + } while (!readFrame); + + return readFrame; + } + + /** + * Returns the timestamp in milliseconds of the currently active frame. That timestamp is based on a best-effort calculation by the FFMPEG decoder. + * + * @return Timestamp of the current frame. + */ + private Long getFrameTimestamp() { + AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); + return Math.floorDiv((this.decodedFrame.best_effort_timestamp() * timebase.num() * 1000), timebase.den()); + } + + /** + * Reads the decoded frame and copies it directly into a new AudioFrame data-structure. + * + * @param samples Number of samples returned by the decoder. + */ + private void readOriginal(int samples) { + /* Allocate output buffer... */ + int buffersize = samples * avutil.av_get_bytes_per_sample(this.decodedFrame.format()) * this.decodedFrame.channels(); + byte[] buffer = new byte[buffersize]; + this.decodedFrame.data(0).position(0).get(buffer); + + /* ... and add frame to queue. */ + this.frameQueue.add(new AudioFrame(this.getFrameNumber(), this.getFrameTimestamp(), buffer, this.descriptor)); + } + + + /** + * Reads the decoded frame and re.samples it using the SWR-CTX. The re-sampled frame is then copied into a AudioFrame data-structure. + * + * @param samples Number of samples returned by the decoder. + */ + private void readResampled(int samples) { + /* Convert decoded frame. Break if resampling fails.*/ + if (swresample.swr_convert(this.swr_ctx, null, 0, this.decodedFrame.data(), samples) < 0) { + LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName()); + return; } - /** - * Returns the timestamp in milliseconds of the currently active frame. That timestamp is based on - * a best-effort calculation by the FFMPEG decoder. - * - * @return Timestamp of the current frame. - */ - private Long getFrameTimestamp() { - AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); - return Math.floorDiv((this.decodedFrame.best_effort_timestamp() * timebase.num() * 1000), timebase.den()); + /* Prepare ByteOutputStream to write resampled data to. */ + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + /* Now drain buffer and write samples to queue. */ + while (true) { + /* Estimate number of samples that were converted. */ + int out_samples = swresample.swr_get_out_samples(this.swr_ctx, 0); + if (out_samples < BYTES_PER_SAMPLE * this.resampledFrame.channels()) { + break; + } + + /* Allocate output frame and read converted samples. If no sample was read -> break (draining completed). */ + avutil.av_samples_alloc(this.resampledFrame.data(), out_linesize, this.resampledFrame.channels(), out_samples, TARGET_FORMAT, 0); + out_samples = swresample.swr_convert(this.swr_ctx, this.resampledFrame.data(), out_samples, null, 0); + if (out_samples == 0) { + break; + } + + /* Allocate output buffer... */ + int buffersize = out_samples * BYTES_PER_SAMPLE * this.resampledFrame.channels(); + byte[] buffer = new byte[buffersize]; + this.resampledFrame.data(0).position(0).get(buffer); + + try { + stream.write(buffer); + } catch (IOException e) { + LOGGER.error("Could not write re-sampled frame to ByteArrayOutputStream due to an exception ({}).", LogHelper.getStackTrace(e)); + break; + } + + avutil.av_freep(this.resampledFrame.data()); } - /** - * Reads the decoded frame and copies it directly into a new AudioFrame data-structure. - * - * @param samples Number of samples returned by the decoder. - */ - private void readOriginal(int samples) { - /* Allocate output buffer... */ - int buffersize = samples * avutil.av_get_bytes_per_sample(this.decodedFrame.format()) * this.decodedFrame.channels(); - byte[] buffer = new byte[buffersize]; - this.decodedFrame.data(0).position(0).get(buffer); - - /* ... and add frame to queue. */ - this.frameQueue.add(new AudioFrame(this.getFrameNumber(), this.getFrameTimestamp(), buffer, this.descriptor)); + /* ... and add frame to queue. */ + this.frameQueue.add(new AudioFrame(this.getFrameNumber(), this.getFrameTimestamp(), stream.toByteArray(), this.descriptor)); + } + + /** + * Seeks to a specific frame, discarding all frames in between. + * + * @param frameNumber The frame number that is being sought. + */ + @Override + public void seekToFrame(int frameNumber) { + while (this.pCodecCtx.frame_number() < frameNumber) { + if (!readFrame(false)) { + break; + } + } + } + + /** + * Returns the current frame number. This methods gets this information directly from the decoder. + * + * @return Frame that is currently being processed. + */ + @Override + public int getFrameNumber() { + return this.pCodecCtx.frame_number(); + } + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + if (!Files.exists(path)) { + LOGGER.error("File does not exist {}", path.toString()); + return false; } + /* Read decoder configuration. */ + int samplerate = decoderConfig.namedAsInt(CONFIG_SAMPLERATE_PROPERTY, CONFIG_SAMPLERATE_DEFAULT); + int channels = decoderConfig.namedAsInt(CONFIG_CHANNELS_PROPERTY, CONFIG_CHANNELS_DEFAULT); + long channellayout = avutil.av_get_default_channel_layout(channels); - /** - * Reads the decoded frame and re.samples it using the SWR-CTX. The re-sampled frame is then - * copied into a AudioFrame data-structure. - * - * @param samples Number of samples returned by the decoder. - */ - private void readResampled(int samples) { - /* Convert decoded frame. Break if resampling fails.*/ - if (swresample.swr_convert(this.swr_ctx, null, 0, this.decodedFrame.data(), samples) < 0) { - LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName()); - return; - } - - /* Prepare ByteOutputStream to write resampled data to. */ - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - - /* Now drain buffer and write samples to queue. */ - while(true) { - /* Estimate number of samples that were converted. */ - int out_samples = swresample.swr_get_out_samples(this.swr_ctx, 0); - if (out_samples < BYTES_PER_SAMPLE * this.resampledFrame.channels()) { - break; - } - - /* Allocate output frame and read converted samples. If no sample was read -> break (draining completed). */ - avutil.av_samples_alloc(this.resampledFrame.data(), out_linesize, this.resampledFrame.channels(), out_samples, TARGET_FORMAT, 0); - out_samples = swresample.swr_convert(this.swr_ctx, this.resampledFrame.data(), out_samples, null, 0); - if (out_samples == 0) { - break; - } - - /* Allocate output buffer... */ - int buffersize = out_samples * BYTES_PER_SAMPLE * this.resampledFrame.channels(); - byte[] buffer = new byte[buffersize]; - this.resampledFrame.data(0).position(0).get(buffer); - - try { - stream.write(buffer); - } catch (IOException e) { - LOGGER.error("Could not write re-sampled frame to ByteArrayOutputStream due to an exception ({}).", LogHelper.getStackTrace(e)); - break; - } - - avutil.av_freep(this.resampledFrame.data()); - } + /* Initialize the AVFormatContext. */ + this.pFormatCtx = avformat.avformat_alloc_context(); - /* ... and add frame to queue. */ - this.frameQueue.add(new AudioFrame(this.getFrameNumber(), this.getFrameTimestamp(), stream.toByteArray(), this.descriptor)); + /* Open file (pure frames or video + frames). */ + if (avformat.avformat_open_input(this.pFormatCtx, path.toString(), null, null) != 0) { + LOGGER.error("Error while accessing file {}.", path.toString()); + return false; } - /** - * Seeks to a specific frame, discarding all frames in between. - * - * @param frameNumber The frame number that is being sought. - */ - @Override - public void seekToFrame(int frameNumber) { - while(this.pCodecCtx.frame_number() < frameNumber){ - if(!readFrame(false)){ - break; - } - } + /* Retrieve stream information. */ + if (avformat.avformat_find_stream_info(this.pFormatCtx, (PointerPointer) null) < 0) { + LOGGER.error("Couldn't find stream information."); + return false; } - /** - * Returns the current frame number. This methods gets this information - * directly from the decoder. - * - * @return Frame that is currently being processed. - */ - @Override - public int getFrameNumber() { - return this.pCodecCtx.frame_number(); + /* Find the best stream. */ + final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); + this.audioStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_AUDIO, -1, -1, codec, 0); + if (this.audioStream == -1) { + LOGGER.error("Couldn't find a supported audio stream."); + return false; } - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - if(!Files.exists(path)){ - LOGGER.error("File does not exist {}", path.toString()); - return false; - } - - /* Read decoder configuration. */ - int samplerate = decoderConfig.namedAsInt(CONFIG_SAMPLERATE_PROPERTY, CONFIG_SAMPLERATE_DEFAULT); - int channels = decoderConfig.namedAsInt(CONFIG_CHANNELS_PROPERTY, CONFIG_CHANNELS_DEFAULT); - long channellayout = avutil.av_get_default_channel_layout(channels); - - /* Initialize the AVFormatContext. */ - this.pFormatCtx = avformat.avformat_alloc_context(); + /* Allocate new codec-context. */ + this.pCodecCtx = avcodec.avcodec_alloc_context3(codec); + avcodec.avcodec_parameters_to_context(this.pCodecCtx, this.pFormatCtx.streams(this.audioStream).codecpar()); - /* Open file (pure frames or video + frames). */ - if (avformat.avformat_open_input(this.pFormatCtx, path.toString(), null, null) != 0) { - LOGGER.error("Error while accessing file {}.", path.toString()); - return false; - } + /* Initialize context with stream's codec settings. */ + this.pCodecCtx.sample_rate(this.pFormatCtx.streams(this.audioStream).codecpar().sample_rate()); + this.pCodecCtx.channels(this.pFormatCtx.streams(this.audioStream).codecpar().channels()); + this.pCodecCtx.channel_layout(this.pFormatCtx.streams(this.audioStream).codecpar().channel_layout()); + this.pCodecCtx.sample_fmt(this.pFormatCtx.streams(this.audioStream).codecpar().format()); - /* Retrieve stream information. */ - if (avformat.avformat_find_stream_info(this.pFormatCtx, (PointerPointer)null) < 0) { - LOGGER.error("Couldn't find stream information."); - return false; - } - - /* Find the best stream. */ - final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); - this.audioStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_AUDIO,-1, -1, codec, 0); - if (this.audioStream == -1) { - LOGGER.error("Couldn't find a supported audio stream."); - return false; - } - - /* Allocate new codec-context. */ - this.pCodecCtx = avcodec.avcodec_alloc_context3(codec); - avcodec.avcodec_parameters_to_context(this.pCodecCtx, this.pFormatCtx.streams(this.audioStream).codecpar()); - - /* Initialize context with stream's codec settings. */ - this.pCodecCtx.sample_rate(this.pFormatCtx.streams(this.audioStream).codecpar().sample_rate()); - this.pCodecCtx.channels(this.pFormatCtx.streams(this.audioStream).codecpar().channels()); - this.pCodecCtx.channel_layout(this.pFormatCtx.streams(this.audioStream).codecpar().channel_layout()); - this.pCodecCtx.sample_fmt(this.pFormatCtx.streams(this.audioStream).codecpar().format()); - - /* Open the code context. */ - if (avcodec.avcodec_open2(this.pCodecCtx, codec, (AVDictionary) null) < 0) { - LOGGER.error("Could not open audio codec."); - return false; - } - - /* Allocate the re-sample context. */ - this.swr_ctx = swresample.swr_alloc_set_opts(null, channellayout, TARGET_FORMAT, samplerate, this.pCodecCtx.channel_layout(), this.pCodecCtx.sample_fmt(), this.pCodecCtx.sample_rate(), 0, null); - if(swresample.swr_init(this.swr_ctx) < 0) { - this.swr_ctx = null; - LOGGER.warn("Could not open re-sample context - original format will be kept!"); - } + /* Open the code context. */ + if (avcodec.avcodec_open2(this.pCodecCtx, codec, (AVDictionary) null) < 0) { + LOGGER.error("Could not open audio codec."); + return false; + } - /* Initialize the packet. */ - this.packet = avcodec.av_packet_alloc(); - if (this.packet == null) { - LOGGER.error("Could not allocate packet data structure for decoded data."); - return false; - } + /* Allocate the re-sample context. */ + this.swr_ctx = swresample.swr_alloc_set_opts(null, channellayout, TARGET_FORMAT, samplerate, this.pCodecCtx.channel_layout(), this.pCodecCtx.sample_fmt(), this.pCodecCtx.sample_rate(), 0, null); + if (swresample.swr_init(this.swr_ctx) < 0) { + this.swr_ctx = null; + LOGGER.warn("Could not open re-sample context - original format will be kept!"); + } - /* Allocate frame that holds decoded frame information. */ - this.decodedFrame = avutil.av_frame_alloc(); - if (this.decodedFrame == null) { - LOGGER.error("Could not allocate frame data structure for decoded data."); - return false; - } + /* Initialize the packet. */ + this.packet = avcodec.av_packet_alloc(); + if (this.packet == null) { + LOGGER.error("Could not allocate packet data structure for decoded data."); + return false; + } - /* Initialize out-frame. */ - this.resampledFrame = avutil.av_frame_alloc(); - if (this.resampledFrame == null) { - LOGGER.error("Could not allocate frame data structure for re-sampled data."); - return false; - } - this.resampledFrame.channel_layout(channellayout); - this.resampledFrame.sample_rate(samplerate); - this.resampledFrame.channels(channels); - this.resampledFrame.format(TARGET_FORMAT); + /* Allocate frame that holds decoded frame information. */ + this.decodedFrame = avutil.av_frame_alloc(); + if (this.decodedFrame == null) { + LOGGER.error("Could not allocate frame data structure for decoded data."); + return false; + } + /* Initialize out-frame. */ + this.resampledFrame = avutil.av_frame_alloc(); + if (this.resampledFrame == null) { + LOGGER.error("Could not allocate frame data structure for re-sampled data."); + return false; + } + this.resampledFrame.channel_layout(channellayout); + this.resampledFrame.sample_rate(samplerate); + this.resampledFrame.channels(channels); + this.resampledFrame.format(TARGET_FORMAT); - /* Initialize the AudioDescriptor. */ - AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); - long duration = Math.floorDiv(1000L * timebase.num() * this.pFormatCtx.streams(this.audioStream).duration(), timebase.den()); - if (this.swr_ctx == null) { - this.descriptor = new AudioDescriptor(this.pCodecCtx.sample_rate(), this.pCodecCtx.channels(), duration); - } else { - this.descriptor = new AudioDescriptor(this.resampledFrame.sample_rate(), this.resampledFrame.channels(), duration); - } + /* Initialize the AudioDescriptor. */ + AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); + long duration = Math.floorDiv(1000L * timebase.num() * this.pFormatCtx.streams(this.audioStream).duration(), timebase.den()); - /* Completed initialization. */ - LOGGER.debug("{} was initialized successfully.", this.getClass().getName()); - return true; + if (this.swr_ctx == null) { + this.descriptor = new AudioDescriptor(this.pCodecCtx.sample_rate(), this.pCodecCtx.channels(), duration); + } else { + this.descriptor = new AudioDescriptor(this.resampledFrame.sample_rate(), this.resampledFrame.channels(), duration); } - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public AudioFrame getNext() { - if(this.frameQueue.isEmpty()){ - boolean frame = readFrame(true); - if (!frame) { - this.complete.set(true); - } - } - return this.frameQueue.poll(); + /* Completed initialization. */ + LOGGER.debug("{} was initialized successfully.", this.getClass().getName()); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public AudioFrame getNext() { + if (this.frameQueue.isEmpty()) { + boolean frame = readFrame(true); + if (!frame) { + this.complete.set(true); + } } - - /** - * Returns the total number of content pieces T this decoder can return for a given file. May be - * zero if the decoder cannot determine that number. - * - * @return Number of frames in the audio-stream (if known). - */ - @Override - public int count() { - return (int) this.pFormatCtx.streams(this.audioStream).nb_frames(); + return this.frameQueue.poll(); + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. May be zero if the decoder cannot determine that number. + * + * @return Number of frames in the audio-stream (if known). + */ + @Override + public int count() { + return (int) this.pFormatCtx.streams(this.audioStream).nb_frames(); + } + + /** + * Indicates whether or not the decoder has more content to return. + * + * @return True if more content can be retrieved, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } + + /** + * Closes the FFMpegAudioDecoder and frees all (non-native) resources associated with it. + */ + @Override + public void close() { + if (this.pFormatCtx == null) { + return; } - /** - * Indicates whether or not the decoder has more content to return. - * - * @return True if more content can be retrieved, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + /* Free the audio frames */ + if (this.decodedFrame != null) { + avutil.av_frame_free(this.decodedFrame); + this.decodedFrame = null; } - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; + if (this.resampledFrame != null) { + avutil.av_frame_free(this.resampledFrame); + this.resampledFrame = null; } - /** - * Closes the FFMpegAudioDecoder and frees all (non-native) resources associated with it. - */ - @Override - public void close() { - if (this.pFormatCtx == null) { - return; - } - - /* Free the audio frames */ - if (this.decodedFrame != null) { - avutil.av_frame_free(this.decodedFrame); - this.decodedFrame = null; - } - - if (this.resampledFrame != null) { - avutil.av_frame_free(this.resampledFrame); - this.resampledFrame = null; - } - - /* Free the packet. */ - if (this.packet != null) { - avcodec.av_packet_free(this.packet); - this.packet = null; - } - - /* Frees the SWR context. */ - if (this.swr_ctx != null) { - swresample.swr_free(this.swr_ctx); - this.swr_ctx = null; - } + /* Free the packet. */ + if (this.packet != null) { + avcodec.av_packet_free(this.packet); + this.packet = null; + } - /* Close the codec context. */ - if (this.pCodecCtx != null) { - avcodec.avcodec_free_context(this.pCodecCtx); - this.pCodecCtx = null; - } + /* Frees the SWR context. */ + if (this.swr_ctx != null) { + swresample.swr_free(this.swr_ctx); + this.swr_ctx = null; + } - /* Close the audio file context. */ - avformat.avformat_close_input(this.pFormatCtx); - this.pFormatCtx = null; + /* Close the codec context. */ + if (this.pCodecCtx != null) { + avcodec.avcodec_free_context(this.pCodecCtx); + this.pCodecCtx = null; } + /* Close the audio file context. */ + avformat.avformat_close_input(this.pFormatCtx); + this.pFormatCtx = null; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Converter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Converter.java index 493132ee8..6d8507e98 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Converter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Converter.java @@ -1,25 +1,25 @@ package org.vitrivr.cineast.core.extraction.decode.general; -import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; - import java.nio.file.Path; import java.util.Set; +import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; public interface Converter { - /** - * Converts a single file to a QueryContainer. - * - * @param path Path the file that should be converted. - * @return QueryContainer for the specified file. - */ - AbstractQueryTermContainer convert(Path path); - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - Set supportedFiles(); + /** + * Converts a single file to a QueryContainer. + * + * @param path Path the file that should be converted. + * @return QueryContainer for the specified file. + */ + AbstractQueryTermContainer convert(Path path); + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + Set supportedFiles(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java index cf053d950..29448df7d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/general/Decoder.java @@ -1,81 +1,72 @@ package org.vitrivr.cineast.core.extraction.decode.general; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.config.CacheConfig; - import java.nio.file.Path; import java.util.Set; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; /** - * General interface for Decoder classes. These classes take a file as input and return one to many - * content items of type T. Depending on what file-type this may be a single image, a set of audio samples, - * video frames etc. - * - * Decoders are initialized with the init() method and closed with the close() method. Some decoders - * may be reusable, which means that they can be re-initialized by calling init() again. Use the reusable() - * method to determine whether or not a Decoder can be re-used. - * + * General interface for Decoder classes. These classes take a file as input and return one to many content items of type T. Depending on what file-type this may be a single image, a set of audio samples, video frames etc. + *

    + * Decoders are initialized with the init() method and closed with the close() method. Some decoders may be reusable, which means that they can be re-initialized by calling init() again. Use the reusable() method to determine whether or not a Decoder can be re-used. */ public interface Decoder extends AutoCloseable { - /** - * Initializes the decoder with a {@link Path}. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * Important: It is not safe to call getNext() of an uninitialized decoder or a Decoder that - * returned false upon initialization. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by the {@link Decoder}. - * @return True if initialization was successful, false otherwise. - */ - boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig); - /** - * Closes the Decoder. This method should cleanup and relinquish all resources. - * - * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - void close(); + /** + * Initializes the decoder with a {@link Path}. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * Important: It is not safe to call getNext() of an uninitialized decoder or a Decoder that + * returned false upon initialization. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by the {@link Decoder}. + * @return True if initialization was successful, false otherwise. + */ + boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig); + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

    + * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + void close(); - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - T getNext(); + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + T getNext(); - /** - * Returns the total number of content pieces T this decoder can return - * for a given file. - */ - int count(); + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + int count(); - /** - * Indicates whether or not the decoder has more content to return. - * - * @return True if more content can be retrieved, false otherwise. - */ - boolean complete(); + /** + * Indicates whether or not the decoder has more content to return. + * + * @return True if more content can be retrieved, false otherwise. + */ + boolean complete(); - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - Set supportedFiles(); + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + Set supportedFiles(); - /** - * Indicates whether or not a particular instance of the Decoder interface can be re-used. If this method returns - * true, it is save to call init() with a new file after the previous file has been fully read. - * - * This property can be leveraged to reduce the memory-footprint of the application. - * - * @return True if re-use is possible, false otherwise. - */ - default boolean canBeReused() { - return false; - } + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used. If this method returns true, it is save to call init() with a new file after the previous file has been fully read. + *

    + * This property can be leveraged to reduce the memory-footprint of the application. + * + * @return True if re-use is possible, false otherwise. + */ + default boolean canBeReused() { + return false; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java index 89ed1d788..3adb89a18 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/DefaultImageDecoder.java @@ -1,13 +1,6 @@ package org.vitrivr.cineast.core.extraction.decode.image; import com.twelvemonkeys.image.ResampleOp; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.config.CacheConfig; -import org.vitrivr.cineast.core.extraction.decode.general.Decoder; - -import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.awt.image.BufferedImageOp; import java.io.IOException; @@ -20,141 +13,152 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import javax.imageio.ImageIO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; public class DefaultImageDecoder implements Decoder { - /* Configuration property-names and defaults for the DefaultImageDecoder. */ - private static final String CONFIG_BOUNDS_PROPERTY = "bounds"; - private static final int CONFIG_BOUNDS_DEFAULT = 1024; - - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** HashSet containing all the mime-types supported by this ImageDecoder instance. */ - private static Set supportedFiles = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ImageIO.getReaderMIMETypes()))); - - /** Bounds used to rescale the image. */ - private int rescale_bounds = CONFIG_BOUNDS_DEFAULT; - - /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** Path to the input file. */ - private Path input; - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.input = path; - this.complete.set(false); - if (decoderConfig != null) { - this.rescale_bounds = decoderConfig.namedAsInt(CONFIG_BOUNDS_PROPERTY, CONFIG_BOUNDS_DEFAULT); - } - return true; + /* Configuration property-names and defaults for the DefaultImageDecoder. */ + private static final String CONFIG_BOUNDS_PROPERTY = "bounds"; + private static final int CONFIG_BOUNDS_DEFAULT = 1024; + + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static Set supportedFiles = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ImageIO.getReaderMIMETypes()))); + + /** + * Bounds used to rescale the image. + */ + private int rescale_bounds = CONFIG_BOUNDS_DEFAULT; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Path to the input file. + */ + private Path input; + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.input = path; + this.complete.set(false); + if (decoderConfig != null) { + this.rescale_bounds = decoderConfig.namedAsInt(CONFIG_BOUNDS_PROPERTY, CONFIG_BOUNDS_DEFAULT); } - - /** - * Obtains and returns a result by decoding the image. The image is re-rescaled to match the - * bounding box defined by RESCALE_BOUNDS. - * - * @return BufferedImage of the decoded image file or null of decoding failed. - */ - @Override - public BufferedImage getNext() { - InputStream is = null; - BufferedImage output = null; - BufferedImage input; - try { - is = Files.newInputStream(this.input, StandardOpenOption.READ); - input = ImageIO.read(is); - - if (input != null) { - int width = input.getWidth(); - int height = input.getHeight(); - float ratio = 0; - - if (width > rescale_bounds) { - ratio = (float)rescale_bounds/(float)width; - width = (int)(width * ratio); - height = (int)(height * ratio); - } - - if (height > rescale_bounds) { - ratio = (float)rescale_bounds/(float)height; - width = (int)(width * ratio); - height = (int)(height * ratio); - } - - final BufferedImageOp resampler = new ResampleOp(width, height, ResampleOp.FILTER_LANCZOS); // A good default filter, see class documentation for more info - output = resampler.filter(input, null); - } - } catch (IOException | IllegalArgumentException e) { - LOGGER.fatal("A severe error occurred while trying to decode the image file under '{}'. Image will be skipped...", this.input.toString()); - } finally { - try { - if (is != null) { - is.close(); - } - } catch (IOException e) { - LOGGER.warn("Could not close the input stream of the image file under {}.", this.input.toString()); - } - this.complete.set(true); + return true; + } + + /** + * Obtains and returns a result by decoding the image. The image is re-rescaled to match the bounding box defined by RESCALE_BOUNDS. + * + * @return BufferedImage of the decoded image file or null of decoding failed. + */ + @Override + public BufferedImage getNext() { + InputStream is = null; + BufferedImage output = null; + BufferedImage input; + try { + is = Files.newInputStream(this.input, StandardOpenOption.READ); + input = ImageIO.read(is); + + if (input != null) { + int width = input.getWidth(); + int height = input.getHeight(); + float ratio = 0; + + if (width > rescale_bounds) { + ratio = (float) rescale_bounds / (float) width; + width = (int) (width * ratio); + height = (int) (height * ratio); } - return output; - } - - /** - * Returns the total number of content pieces T this decoder can return - * for a given file. - */ - @Override - public int count() { - return 1; - } - /** - * Returns a list of supported files. - */ - @Override - public Set supportedFiles() { - return supportedFiles; - } - - /** - * Indicates whether or not a particular instance of the Decoder interface can - * be re-used or not. This property can be leveraged to reduce the memory-footpring - * of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; - } + if (height > rescale_bounds) { + ratio = (float) rescale_bounds / (float) height; + width = (int) (width * ratio); + height = (int) (height * ratio); + } - /** - * Indicates whether or not the current decoder instance is complete i.e. if there is - * content left that can be obtained. - * - * @return true if there is still content, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + final BufferedImageOp resampler = new ResampleOp(width, height, ResampleOp.FILTER_LANCZOS); // A good default filter, see class documentation for more info + output = resampler.filter(input, null); + } + } catch (IOException | IllegalArgumentException e) { + LOGGER.fatal("A severe error occurred while trying to decode the image file under '{}'. Image will be skipped...", this.input.toString()); + } finally { + try { + if (is != null) { + is.close(); + } + } catch (IOException e) { + LOGGER.warn("Could not close the input stream of the image file under {}.", this.input.toString()); + } + this.complete.set(true); } - - /** - * Nothing to close! - */ - @Override - public void close() {} + return output; + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; + } + + /** + * Returns a list of supported files. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Nothing to close! + */ + @Override + public void close() { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java index 421b82ca4..783b8da6b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequence.java @@ -1,13 +1,6 @@ package org.vitrivr.cineast.core.extraction.decode.image; import com.twelvemonkeys.image.ResampleOp; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.data.MediaType; -import org.vitrivr.cineast.core.data.Pair; - -import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.awt.image.BufferedImageOp; import java.io.IOException; @@ -15,92 +8,109 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Supplier; +import javax.imageio.ImageIO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.MediaType; +import org.vitrivr.cineast.core.data.Pair; /** - * Represents a media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a folder. - * This class is merely an internal abstraction of that type and the content it represents. Its sole purpose is to - * provide lazy access to the images contained in such a sequence during the extraction process. - * + * Represents a media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a folder. This class is merely an internal abstraction of that type and the content it represents. Its sole purpose is to provide lazy access to the images contained in such a sequence during the extraction process. */ public final class ImageSequence { - /* Configuration property-names and defaults for the DefaultImageDecoder. */ - private static final String CONFIG_BOUNDS_PROPERTY = "bounds"; - private static final int CONFIG_BOUNDS_DEFAULT = 1024; - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); + /* Configuration property-names and defaults for the DefaultImageDecoder. */ + private static final String CONFIG_BOUNDS_PROPERTY = "bounds"; + private static final int CONFIG_BOUNDS_DEFAULT = 1024; - /** HashSet containing all the mime-types supported by this ImageDecoder instance. */ - public static final Set SUPPORTED_FILES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ImageIO.getReaderMIMETypes()))); + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); - /** Bounds used to rescale the image. */ - private final int rescale_bounds; + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + public static final Set SUPPORTED_FILES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ImageIO.getReaderMIMETypes()))); - /** List of suppliers for {@link BufferedImage}s. */ - private final Queue>>> images = new ConcurrentLinkedQueue<>(); + /** + * Bounds used to rescale the image. + */ + private final int rescale_bounds; - /** - * Default constructor for {@link ImageSequence}. - * - * @param config {@link DecoderConfig} to use. - */ - public ImageSequence(DecoderConfig config) { - if (config != null) { - this.rescale_bounds = config.namedAsInt(CONFIG_BOUNDS_PROPERTY, CONFIG_BOUNDS_DEFAULT); - } else { - this.rescale_bounds = CONFIG_BOUNDS_DEFAULT; - } - } + /** + * List of suppliers for {@link BufferedImage}s. + */ + private final Queue>>> images = new ConcurrentLinkedQueue<>(); - /** - * Adds a new {@link Path} to an image to this {@link ImageSequence} - * - * @param path The {@link Path} to add. - */ - public void add(Path path) { - this.images.add(() -> { - try (final InputStream is = Files.newInputStream(path, StandardOpenOption.READ);){ - final BufferedImage input = ImageIO.read(is); - if (input != null) { - int width = input.getWidth(); - int height = input.getHeight(); - float ratio; + /** + * Default constructor for {@link ImageSequence}. + * + * @param config {@link DecoderConfig} to use. + */ + public ImageSequence(DecoderConfig config) { + if (config != null) { + this.rescale_bounds = config.namedAsInt(CONFIG_BOUNDS_PROPERTY, CONFIG_BOUNDS_DEFAULT); + } else { + this.rescale_bounds = CONFIG_BOUNDS_DEFAULT; + } + } - if (width > rescale_bounds) { - ratio = (float)rescale_bounds/(float)width; - width = (int)(width * ratio); - height = (int)(height * ratio); - } + /** + * Adds a new {@link Path} to an image to this {@link ImageSequence} + * + * @param path The {@link Path} to add. + */ + public void add(Path path) { + this.images.add(() -> { + try (final InputStream is = Files.newInputStream(path, StandardOpenOption.READ);) { + final BufferedImage input = ImageIO.read(is); + if (input != null) { + int width = input.getWidth(); + int height = input.getHeight(); + float ratio; - if (height > rescale_bounds) { - ratio = (float)rescale_bounds/(float)height; - width = (int)(width * ratio); - height = (int)(height * ratio); - } + if (width > rescale_bounds) { + ratio = (float) rescale_bounds / (float) width; + width = (int) (width * ratio); + height = (int) (height * ratio); + } - final BufferedImageOp resampler = new ResampleOp(width, height, ResampleOp.FILTER_LANCZOS); // A good default filter, see class documentation for more info - return new Pair<>(path, Optional.of(resampler.filter(input, null))); - } - } catch (IOException | IllegalArgumentException e) { - LOGGER.fatal("A severe error occurred while trying to decode the image file under '{}'. Image will be skipped...", path.toString()); - } - return new Pair<>(path, Optional.empty()); - }); - } + if (height > rescale_bounds) { + ratio = (float) rescale_bounds / (float) height; + width = (int) (width * ratio); + height = (int) (height * ratio); + } - /** - * Pops and returns the next {@link BufferedImage}. Since the {@link BufferedImage}s are calculated lazily, invocation of this method can take a while. - * - * @return Next {@link BufferedImage} - */ - public Pair> pop() { - final Supplier>> supplier = this.images.poll(); - if (supplier != null) { - return supplier.get(); + final BufferedImageOp resampler = new ResampleOp(width, height, ResampleOp.FILTER_LANCZOS); // A good default filter, see class documentation for more info + return new Pair<>(path, Optional.of(resampler.filter(input, null))); } - return null; + } catch (IOException | IllegalArgumentException e) { + LOGGER.fatal("A severe error occurred while trying to decode the image file under '{}'. Image will be skipped...", path.toString()); + } + return new Pair<>(path, Optional.empty()); + }); + } + + /** + * Pops and returns the next {@link BufferedImage}. Since the {@link BufferedImage}s are calculated lazily, invocation of this method can take a while. + * + * @return Next {@link BufferedImage} + */ + public Pair> pop() { + final Supplier>> supplier = this.images.poll(); + if (supplier != null) { + return supplier.get(); } + return null; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java index 9597db52e..bbf56066c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/image/ImageSequenceDecoder.java @@ -1,60 +1,65 @@ package org.vitrivr.cineast.core.extraction.decode.image; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.util.MimeTypeHelper; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; - /** - * Decoder for media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a single - * folder that, in terms of Cineast's data model, belong together. + * Decoder for media object of type {@link MediaType#IMAGE_SEQUENCE}, i.e. a sequence of images contained in a single folder that, in terms of Cineast's data model, belong together. * * Important: Unlike other implementations of {@link Decoder} this class operates on folders only! - * It assumes that the images belonging to the sequence are contained in that folder (no subfolders) and that the images' - * occurrence in the sequence correspond to the natural ordering of the filenames in ascending direction. + * It assumes that the images belonging to the sequence are contained in that folder (no subfolders) and that the images' occurrence in the sequence correspond to the natural ordering of the filenames in ascending direction. * * @see ImageSequence - * */ public class ImageSequenceDecoder implements Decoder { - /** Default logging facility. */ + /** + * Default logging facility. + */ private static final Logger LOGGER = LogManager.getLogger(); private static final Set SUPPORTED = new HashSet<>(); + static { SUPPORTED.add("application/octet-stream"); } private final DirectoryStream.Filter filter = file -> Files.isRegularFile(file) && ImageSequence.SUPPORTED_FILES.contains(MimeTypeHelper.getContentType(file)); - /** Path to the folder that contains the next {@link ImageSequence}. */ + /** + * Path to the folder that contains the next {@link ImageSequence}. + */ private Path path; - /** {@link DecoderConfig} instance to use. */ + /** + * {@link DecoderConfig} instance to use. + */ private DecoderConfig decoderConfig; - /** {@link CacheConfig} instance to use. */ + /** + * {@link CacheConfig} instance to use. + */ private CacheConfig cacheConfig; /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. * - * @param path Path to the file that should be decoded. + * @param path Path to the file that should be decoded. * @param decoderConfig {@link DecoderConfig} used by this {@link ImageSequenceDecoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link ImageSequenceDecoder} + * @param cacheConfig The {@link CacheConfig} used by this {@link ImageSequenceDecoder} * @return True if initialization was successful, false otherwise. */ @Override @@ -66,7 +71,8 @@ public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheCon } @Override - public void close() { } + public void close() { + } @Override public ImageSequence getNext() { @@ -76,13 +82,13 @@ public ImageSequence getNext() { final ImageSequence sequence = new ImageSequence(this.decoderConfig); if (this.path != null) { if (Files.isDirectory(path)) { - try (final DirectoryStream directoryStream = Files.newDirectoryStream(path, this.filter)){ + try (final DirectoryStream directoryStream = Files.newDirectoryStream(path, this.filter)) { final LinkedList paths = new LinkedList<>(); - for (Path p: directoryStream){ + for (Path p : directoryStream) { paths.add(p); } paths.sort(Comparator.comparing(Path::getFileName)); - for (Path p: paths){ + for (Path p : paths) { sequence.add(p); } } catch (IOException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoder.java index 496559a84..cf389f05a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoder.java @@ -4,9 +4,7 @@ import org.vitrivr.cineast.core.extraction.decode.general.Decoder; /** - * Interface used by all mesh decoders. Those decoders translate arbitrary mesh formats, be they in memory - * or on disk, into a Mesh object that can be used by Cineast. - * + * Interface used by all mesh decoders. Those decoders translate arbitrary mesh formats, be they in memory or on disk, into a Mesh object that can be used by Cineast. */ public interface MeshDecoder extends Decoder { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoderException.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoderException.java index 659e4097b..55069e2f8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoderException.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/MeshDecoderException.java @@ -5,7 +5,7 @@ public class MeshDecoderException extends Exception { private static final long serialVersionUID = -6496491964433759446L; - MeshDecoderException(String message, Throwable cause) { - super(message,cause); - } + MeshDecoderException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java index 3f844e460..addfe645c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularMeshDecoder.java @@ -1,203 +1,207 @@ package org.vitrivr.cineast.core.extraction.decode.m3d; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.data.m3d.Mesh; - -import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; import org.vitrivr.cineast.core.extraction.decode.general.Converter; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.util.MimeTypeHelper; -import java.nio.file.Path; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - /** - * - * Tries to decode arbitrary 3D model files by selecting the correct decoder based on the - * model-file's mime-type. - * + * Tries to decode arbitrary 3D model files by selecting the correct decoder based on the model-file's mime-type. */ public class ModularMeshDecoder implements MeshDecoder, Converter { - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** HashSet containing all the mime-types supported by this ImageDecoder instance. - * - * Important: The decoderForContenttype() method must return a Decoder instance - * for all mime-types contained in this set! */ - private final static Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("application/3d-stl"); - tmp.add("application/3d-obj"); - tmp.add("application/3d-off"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - - /** HashMap containing cached decoder instances */ - private HashMap> cachedDecoders = new HashMap<>(); - - /** Path to the input file. */ - private Path inputFile; - - /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.inputFile = path; - this.complete.set(false); - return true; - } - - /** - * Closes the Decoder. This method should cleanup and relinquish all resources. - *

    - * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - public void close() {} - - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public Mesh getNext() { - final String contenttype = MimeTypeHelper.getContentType(this.inputFile.toFile()); - - /* Try to detach decoder from the list of cached decoders. */ - Decoder decoder = this.cachedDecoders.get(contenttype); - - /* If decoder is null, create a new one. */ - if (decoder == null) { - decoder = decoderForContenttype(contenttype); - } - - /* If decoder is still null, return an emtpy Mesh. */ - if (decoder == null) { - LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); - return Mesh.EMPTY; - } else { - this.cachedDecoders.put(contenttype, decoder); - } - - /* Initialize the decoder and return the decoded mesh. */ - decoder.init(this.inputFile, null, null); - Mesh mesh = decoder.getNext(); - this.complete.set(true); - return mesh; - } - - /** - * Converts a single file to a QueryContainer. - * - * @param path Path the file that should be converted. - * @return QueryContainer for the specified file. - */ - @Override - public AbstractQueryTermContainer convert(Path path) { - final String contenttype = MimeTypeHelper.getContentType(path.toFile()); - - /* Try to detach decoder from the list of cached decoders. */ - Decoder decoder = this.cachedDecoders.get(contenttype); - - /* If decoder is null, create a new one. */ - if (decoder == null) { - decoder = decoderForContenttype(contenttype); - } - - /* If decoder is still null, return an emtpy Mesh. */ - if (decoder == null) { - LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); - return null; - } else { - this.cachedDecoders.put(contenttype, decoder); - } - - /* Initialize the decoder and return the decoded mesh. */ - decoder.init(path, null, null); - Mesh mesh = decoder.getNext(); - return new ModelQueryTermContainer(mesh); - } - /** - * Returns the total number of content pieces T this decoder can return - * for a given file. - */ - @Override - public int count() { - return 0; + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + * + * Important: The decoderForContenttype() method must return a Decoder instance + * for all mime-types contained in this set! + */ + private final static Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3d-stl"); + tmp.add("application/3d-obj"); + tmp.add("application/3d-off"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * HashMap containing cached decoder instances + */ + private HashMap> cachedDecoders = new HashMap<>(); + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

    + * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Mesh getNext() { + final String contenttype = MimeTypeHelper.getContentType(this.inputFile.toFile()); + + /* Try to detach decoder from the list of cached decoders. */ + Decoder decoder = this.cachedDecoders.get(contenttype); + + /* If decoder is null, create a new one. */ + if (decoder == null) { + decoder = decoderForContenttype(contenttype); } - /** - * Indicates whether or not the decoder has more content to return. - * - * @return True if more content can be retrieved, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + /* If decoder is still null, return an emtpy Mesh. */ + if (decoder == null) { + LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); + return Mesh.EMPTY; + } else { + this.cachedDecoders.put(contenttype, decoder); } - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; + /* Initialize the decoder and return the decoded mesh. */ + decoder.init(this.inputFile, null, null); + Mesh mesh = decoder.getNext(); + this.complete.set(true); + return mesh; + } + + /** + * Converts a single file to a QueryContainer. + * + * @param path Path the file that should be converted. + * @return QueryContainer for the specified file. + */ + @Override + public AbstractQueryTermContainer convert(Path path) { + final String contenttype = MimeTypeHelper.getContentType(path.toFile()); + + /* Try to detach decoder from the list of cached decoders. */ + Decoder decoder = this.cachedDecoders.get(contenttype); + + /* If decoder is null, create a new one. */ + if (decoder == null) { + decoder = decoderForContenttype(contenttype); } - /** - * Indicates whether or not a particular instance of the Decoder interface can be re-used. If this method returns - * true, it is save to call init() with a new file after the previous file has been fully read. - *

    - * This property can be leveraged to reduce the memory-footprint of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; + /* If decoder is still null, return an emtpy Mesh. */ + if (decoder == null) { + LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); + return null; + } else { + this.cachedDecoders.put(contenttype, decoder); } - /** - * Selects a Decoder implementation based on the provided content type. - * - * @param contenttype Mime-type for which to select a decoder. - * @return Decoder or null if the mime-type is not supported. - */ - private Decoder decoderForContenttype(String contenttype) { - switch (contenttype) { - case "application/3d-obj": - return new OBJMeshDecoder(); - case "application/3d-stl": - return new STLMeshDecoder(); - case "application/3d-off": - return new OFFMeshDecoder(); - default: - return null; - } + /* Initialize the decoder and return the decoded mesh. */ + decoder.init(path, null, null); + Mesh mesh = decoder.getNext(); + return new ModelQueryTermContainer(mesh); + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 0; + } + + /** + * Indicates whether or not the decoder has more content to return. + * + * @return True if more content can be retrieved, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used. If this method returns true, it is save to call init() with a new file after the previous file has been fully read. + *

    + * This property can be leveraged to reduce the memory-footprint of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Selects a Decoder implementation based on the provided content type. + * + * @param contenttype Mime-type for which to select a decoder. + * @return Decoder or null if the mime-type is not supported. + */ + private Decoder decoderForContenttype(String contenttype) { + switch (contenttype) { + case "application/3d-obj": + return new OBJMeshDecoder(); + case "application/3d-stl": + return new STLMeshDecoder(); + case "application/3d-off": + return new OFFMeshDecoder(); + default: + return null; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java index 384f9dfc4..23bbdff52 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OBJMeshDecoder.java @@ -1,16 +1,5 @@ package org.vitrivr.cineast.core.extraction.decode.m3d; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.joml.Vector3f; -import org.joml.Vector3i; -import org.joml.Vector4i; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.config.CacheConfig; -import org.vitrivr.cineast.core.data.m3d.Mesh; -import org.vitrivr.cineast.core.extraction.decode.general.Decoder; -import org.vitrivr.cineast.core.util.LogHelper; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -21,152 +10,167 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.joml.Vector3i; +import org.joml.Vector4i; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.m3d.Mesh; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.util.LogHelper; /** * Decodes Wavefront OBJ (.obj) files and returns a Mesh representation. Requires JOML to work properly. - * + *

    * Texture information is currently discarded! - * */ public class OBJMeshDecoder implements Decoder { - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** HashSet containing all the mime-types supported by this ImageDecoder instance. */ - private static final Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("application/3d-obj"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - /** Path to the input file. */ - private Path inputFile; - - /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.inputFile = path; - this.complete.set(false); - return true; - } + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public Mesh getNext() { - Mesh mesh = new Mesh(100,100); - try { - InputStream is = Files.newInputStream(this.inputFile); - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String line = null; - while ((line = br.readLine()) != null) { - String[] tokens = line.split("\\s+"); - switch (tokens[0]) { - case "v": - mesh.addVertex(new Vector3f(Float.parseFloat(tokens[1]),Float.parseFloat(tokens[2]),Float.parseFloat(tokens[3]))); - break; - case "f": - boolean quad = (tokens.length == 5); - String[] p1 = tokens[1].split("/"); - String[] p2 = tokens[2].split("/"); - String[] p3 = tokens[3].split("/"); - if (quad) { - String[] p4 = tokens[4].split("/"); - Vector4i vertexIndex = new Vector4i(Integer.parseInt(p1[0])-1,Integer.parseInt(p2[0])-1,Integer.parseInt(p3[0])-1,Integer.parseInt(p4[0])-1); - if (!mesh.addFace(vertexIndex)) { - LOGGER.warn("Could not add face {}/{}/{}/{} because index points to non-existing vertex.",vertexIndex.x, vertexIndex.y, vertexIndex.z, vertexIndex.w); - } - } else { - Vector3i vertexIndex = new Vector3i(Integer.parseInt(p1[0])-1,Integer.parseInt(p2[0])-1,Integer.parseInt(p3[0])-1); - if (!mesh.addFace(vertexIndex)) { - LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.",vertexIndex.x, vertexIndex.y, vertexIndex.z); - } - } - break; - default: - break; - } - } + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3d-obj"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); - br.close(); /* Closes the input stream. */ - } catch (IOException e) { - LOGGER.error("Could not decode OBJ file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e)); - mesh = null; - } catch (NumberFormatException e) { - LOGGER.error("Could not decode OBJ file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString()); - mesh = null; - } catch (ArrayIndexOutOfBoundsException e) { - LOGGER.error("Could not decode OBJ file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); - mesh = null; - } finally { - this.complete.set(true); + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Mesh getNext() { + Mesh mesh = new Mesh(100, 100); + try { + InputStream is = Files.newInputStream(this.inputFile); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line = null; + while ((line = br.readLine()) != null) { + String[] tokens = line.split("\\s+"); + switch (tokens[0]) { + case "v": + mesh.addVertex(new Vector3f(Float.parseFloat(tokens[1]), Float.parseFloat(tokens[2]), Float.parseFloat(tokens[3]))); + break; + case "f": + boolean quad = (tokens.length == 5); + String[] p1 = tokens[1].split("/"); + String[] p2 = tokens[2].split("/"); + String[] p3 = tokens[3].split("/"); + if (quad) { + String[] p4 = tokens[4].split("/"); + Vector4i vertexIndex = new Vector4i(Integer.parseInt(p1[0]) - 1, Integer.parseInt(p2[0]) - 1, Integer.parseInt(p3[0]) - 1, Integer.parseInt(p4[0]) - 1); + if (!mesh.addFace(vertexIndex)) { + LOGGER.warn("Could not add face {}/{}/{}/{} because index points to non-existing vertex.", vertexIndex.x, vertexIndex.y, vertexIndex.z, vertexIndex.w); + } + } else { + Vector3i vertexIndex = new Vector3i(Integer.parseInt(p1[0]) - 1, Integer.parseInt(p2[0]) - 1, Integer.parseInt(p3[0]) - 1); + if (!mesh.addFace(vertexIndex)) { + LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.", vertexIndex.x, vertexIndex.y, vertexIndex.z); + } + } + break; + default: + break; } + } - return mesh; + br.close(); /* Closes the input stream. */ + } catch (IOException e) { + LOGGER.error("Could not decode OBJ file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e)); + mesh = null; + } catch (NumberFormatException e) { + LOGGER.error("Could not decode OBJ file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString()); + mesh = null; + } catch (ArrayIndexOutOfBoundsException e) { + LOGGER.error("Could not decode OBJ file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); + mesh = null; + } finally { + this.complete.set(true); } - /** - * Returns the total number of content pieces T this decoder can return for a given file. - */ - @Override - public int count() { - return 1; - } + return mesh; + } - /** - * Closes the Decoder. This method should cleanup and relinquish all resources. - *

    - * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - public void close() {} - - /** - * Indicates whether or not a particular instance of the Decoder interface can - * be re-used or not. This property can be leveraged to reduce the memory-footpring - * of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; - } + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; + } - /** - * Indicates whether or not the current decoder instance is complete i.e. if there is - * content left that can be obtained. - * - * @return true if there is still content, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); - } + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

    + * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; - } + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java index b529b2f4a..c146365ee 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/OFFMeshDecoder.java @@ -1,16 +1,5 @@ package org.vitrivr.cineast.core.extraction.decode.m3d; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.joml.Vector3f; -import org.joml.Vector3i; -import org.joml.Vector4i; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.config.CacheConfig; -import org.vitrivr.cineast.core.data.m3d.Mesh; -import org.vitrivr.cineast.core.extraction.decode.general.Decoder; -import org.vitrivr.cineast.core.util.LogHelper; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -21,188 +10,206 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.joml.Vector3i; +import org.joml.Vector4i; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.m3d.Mesh; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.util.LogHelper; /** * Decodes OFF (.off) files as defined by [1] and returns a Mesh representation. Requires JOML to work properly. - * + *

    * The OFF format is used by the princeton shape benchmark (PSB) [2] - * + *

    * [1] http://shape.cs.princeton.edu/benchmark/documentation/off_format.html - * - * [2] Philip Shilane, Patrick Min, Michael Kazhdan, and Thomas Funkhouser The Princeton Shape Benchmark - * Shape Modeling International, Genova, Italy, June 2004 - * + *

    + * [2] Philip Shilane, Patrick Min, Michael Kazhdan, and Thomas Funkhouser The Princeton Shape Benchmark Shape Modeling International, Genova, Italy, June 2004 */ public class OFFMeshDecoder implements Decoder { - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Delimiter used to separate two entries. */ - private static final String DELIMITER = " "; - - /** Token used to denote the beginning of the OFF file. */ - private static final String TOKEN_BOF = "OFF"; - - /** HashSet containing all the mime-types supported by this ImageDecoder instance. */ - private static final Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("application/3d-off"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - /** Path to the input file. */ - private Path inputFile; - - /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.inputFile = path; - this.complete.set(false); - return true; + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Delimiter used to separate two entries. + */ + private static final String DELIMITER = " "; + + /** + * Token used to denote the beginning of the OFF file. + */ + private static final String TOKEN_BOF = "OFF"; + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3d-off"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Mesh getNext() { + try { + InputStream is = Files.newInputStream(this.inputFile); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + + /* First line must start with OFF. */ + String line = br.readLine(); + String[] tokens = null; + if (line == null || !line.startsWith(TOKEN_BOF)) { + return null; + } + + /* Now read second line which should contain the number of vertices and faces. */ + line = br.readLine(); + if (line == null) { + return null; + } + tokens = line.split(DELIMITER); + + int vertices = Integer.parseInt(tokens[0]); + int faces = Integer.parseInt(tokens[1]); + + /* Prepare empty mesh. */ + Mesh mesh = new Mesh(faces, vertices); + + /* Now read all the vertices. */ + for (int v = 0; v < vertices; v++) { + line = br.readLine(); + if (line == null) { + LOGGER.error("Could not decode OFF file {} because file seems to be missing some vertices ({}/{}).", this.inputFile.toString(), v, vertices); + return null; } - - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public Mesh getNext() { - try { - InputStream is = Files.newInputStream(this.inputFile); - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - - /* First line must start with OFF. */ - String line = br.readLine(); - String[] tokens = null; - if (line == null || !line.startsWith(TOKEN_BOF)) { - return null; - } - - /* Now read second line which should contain the number of vertices and faces. */ - line = br.readLine(); - if (line == null) { - return null; - } - tokens = line.split(DELIMITER); - - int vertices = Integer.parseInt(tokens[0]); - int faces = Integer.parseInt(tokens[1]); - - /* Prepare empty mesh. */ - Mesh mesh = new Mesh(faces, vertices); - - /* Now read all the vertices. */ - for (int v=0; v - * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - public void close() {} - - /** - * Indicates whether or not a particular instance of the Decoder interface can - * be re-used or not. This property can be leveraged to reduce the memory-footpring - * of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; - } - - /** - * Indicates whether or not the current decoder instance is complete i.e. if there is - * content left that can be obtained. - * - * @return true if there is still content, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + tokens = line.split(DELIMITER); + mesh.addVertex(new Vector3f(Float.parseFloat(tokens[0]), Float.parseFloat(tokens[1]), Float.parseFloat(tokens[2]))); + } + + /* Now read all the faces. */ + for (int f = 0; f < faces; f++) { + line = br.readLine(); + if (line == null) { + LOGGER.error("Could not decode OFF file {} because file seems to be missing some faces ({}/{}).", this.inputFile.toString(), f, vertices); + return null; } - - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; + tokens = line.split(DELIMITER); + if (Integer.parseInt(tokens[0]) == 4) { + Vector4i vertexIndex = new Vector4i(Integer.parseInt(tokens[1]), Integer.parseInt(tokens[2]), Integer.parseInt(tokens[3]), Integer.parseInt(tokens[4])); + if (!mesh.addFace(vertexIndex)) { + LOGGER.warn("Could not add face {}/{}/{}/{} because index points to non-existing vertex.", vertexIndex.x, vertexIndex.y, vertexIndex.z, vertexIndex.w); + } + } else if (Integer.parseInt(tokens[0]) == 3) { + Vector3i vertexIndex = new Vector3i(Integer.parseInt(tokens[1]), Integer.parseInt(tokens[2]), Integer.parseInt(tokens[3])); + if (!mesh.addFace(vertexIndex)) { + LOGGER.warn("Could not add face {}/{}/{}/{} because index points to non-existing vertex.", vertexIndex.x, vertexIndex.y, vertexIndex.z); + } + } else { + LOGGER.error("Could not decode OFF file {} because this implementation of Mesh only supports triangular and quadrilateral faces. The provided number of faces is {}.", this.inputFile.toString(), Integer.parseInt(tokens[0])); + return null; } + } + + br.close(); /* Closes the input stream. */ + + return mesh; + } catch (IOException e) { + LOGGER.error("Could not decode OFF file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e)); + return null; + } catch (NumberFormatException e) { + LOGGER.error("Could not decode OFF file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString(), LogHelper.getStackTrace(e)); + return null; + } catch (ArrayIndexOutOfBoundsException e) { + LOGGER.error("Could not decode OFF file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString(), LogHelper.getStackTrace(e)); + return null; + } finally { + this.complete.set(true); + } + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; + } + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

    + * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java index a6f03e041..b47761748 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/STLMeshDecoder.java @@ -1,16 +1,6 @@ package org.vitrivr.cineast.core.extraction.decode.m3d; import gnu.trove.map.hash.TObjectIntHashMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.joml.Vector3f; -import org.joml.Vector3i; -import org.vitrivr.cineast.core.config.DecoderConfig; -import org.vitrivr.cineast.core.config.CacheConfig; -import org.vitrivr.cineast.core.data.m3d.Mesh; -import org.vitrivr.cineast.core.extraction.decode.general.Decoder; -import org.vitrivr.cineast.core.util.LogHelper; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -23,270 +13,286 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.joml.Vector3i; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.m3d.Mesh; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.util.LogHelper; /** * Decodes STereoLithography (.stl) files and returns a Mesh representation. Requires JOML to work properly. - * */ public class STLMeshDecoder implements Decoder { - /** Default logging facility. */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Maximum number of triangles in a STL file. Larger files are discarded. */ - private static final int MAX_TRIANGLES = 5000000; - - /** HashSet containing all the mime-types supported by this ImageDecoder instance. */ - private static final Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("application/3d-stl"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - /** Path to the input file. */ - private Path inputFile; - - /** Flag indicating whether or not the Decoder is done decoding and the content has been obtained. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.inputFile = path; - this.complete.set(false); - return true; - } - - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public Mesh getNext() { - try { - InputStream is = Files.newInputStream(this.inputFile); - byte[] header = new byte[6]; - if (is.read(header) == 6) { - if ((new String(header)).contains("solid ")) { - LOGGER.info("Found term 'solid' in header. Treating the STL file as ASCII STL file!"); - return this.readAscii(is); - } else { - LOGGER.info("Did not find term 'solid' in header. Treating the STL file as binary STL file!"); - return this.readBinary(is, 74); - } - } else { - LOGGER.warn("Could not read the first 10 bytes of the file {}. This is not a valid STL file.", this.inputFile.toString()); - return null; - } - } catch (IOException e) { - LOGGER.error("Could not decode STL file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e)); - this.complete.set(true); - return null; - } finally { - this.complete.set(true); + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Maximum number of triangles in a STL file. Larger files are discarded. + */ + private static final int MAX_TRIANGLES = 5000000; + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3d-stl"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Mesh getNext() { + try { + InputStream is = Files.newInputStream(this.inputFile); + byte[] header = new byte[6]; + if (is.read(header) == 6) { + if ((new String(header)).contains("solid ")) { + LOGGER.info("Found term 'solid' in header. Treating the STL file as ASCII STL file!"); + return this.readAscii(is); + } else { + LOGGER.info("Did not find term 'solid' in header. Treating the STL file as binary STL file!"); + return this.readBinary(is, 74); } + } else { + LOGGER.warn("Could not read the first 10 bytes of the file {}. This is not a valid STL file.", this.inputFile.toString()); + return null; + } + } catch (IOException e) { + LOGGER.error("Could not decode STL file {} due to an IO exception ({})", this.inputFile.toString(), LogHelper.getStackTrace(e)); + this.complete.set(true); + return null; + } finally { + this.complete.set(true); } - - /** - * Reads an ASCII STL file. - * - * @param is InputStream to read from. - * @return Mesh - * @throws IOException If an error occurs during reading. - */ - private Mesh readAscii(InputStream is) throws IOException { - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String line = null; - - /* Prepare empty mesh. */ - Mesh mesh = new Mesh(100,100); - - /* Prepare helper structures. */ - TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); - int index = 0; - int[] vertexindices = new int[3]; - - while ((line = br.readLine()) != null && !line.startsWith("endsolid")) { - line = line.trim(); - - /* Detect end of STL file. */ - if (line.startsWith("endsolid")) { - break; - } - - /* Detect begin of facet. */ - if (line.startsWith("facet normal ")) { - int vidx = 0; - - while ((line = br.readLine()) != null) { - - line = line.trim(); /* Trim line. */ - - /* Detect end of facet. */ - if (line.equals("endfacet")) { - break; - } - - /* Detect vertex. */ - if (line.startsWith("vertex")) { - String[] splitVertex = line.split("\\s+"); - Vector3f vertex = new Vector3f(Float.parseFloat(splitVertex[1]),Float.parseFloat(splitVertex[2]), Float.parseFloat(splitVertex[3])); - if (!vertexBuffer.containsKey(vertex)) { - mesh.addVertex(vertex); - vertexBuffer.put(vertex, index); - index++; - } - vertexindices[vidx] = vertexBuffer.get(vertex); - vidx++; - } - } - - /* Add a new face to the Mesh. */ - mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2])); + } + + /** + * Reads an ASCII STL file. + * + * @param is InputStream to read from. + * @return Mesh + * @throws IOException If an error occurs during reading. + */ + private Mesh readAscii(InputStream is) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line = null; + + /* Prepare empty mesh. */ + Mesh mesh = new Mesh(100, 100); + + /* Prepare helper structures. */ + TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); + int index = 0; + int[] vertexindices = new int[3]; + + while ((line = br.readLine()) != null && !line.startsWith("endsolid")) { + line = line.trim(); + + /* Detect end of STL file. */ + if (line.startsWith("endsolid")) { + break; + } + + /* Detect begin of facet. */ + if (line.startsWith("facet normal ")) { + int vidx = 0; + + while ((line = br.readLine()) != null) { + + line = line.trim(); /* Trim line. */ + + /* Detect end of facet. */ + if (line.equals("endfacet")) { + break; + } + + /* Detect vertex. */ + if (line.startsWith("vertex")) { + String[] splitVertex = line.split("\\s+"); + Vector3f vertex = new Vector3f(Float.parseFloat(splitVertex[1]), Float.parseFloat(splitVertex[2]), Float.parseFloat(splitVertex[3])); + if (!vertexBuffer.containsKey(vertex)) { + mesh.addVertex(vertex); + vertexBuffer.put(vertex, index); + index++; } + vertexindices[vidx] = vertexBuffer.get(vertex); + vidx++; + } } - /* Close the buffered reader. */ - br.close(); - - /* This covers the case, where the file starts with 'solid ' but is not an ASCII file. Unfortunately, such files do exist. */ - if (mesh.numberOfVertices() == 0) { - LOGGER.warn("The provided ASCII STL file does not seem to contain any normals or vertices. Trying to decode it as binary STL even though it was marked as being ASCII."); - InputStream newIs = Files.newInputStream(this.inputFile); - return this.readBinary(newIs, 80); - } else { - return mesh; - } + /* Add a new face to the Mesh. */ + mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2])); + } } - /** - * Reads a binary STL file. - * - * @param is InputStream to read from. - * @param skip Number of bytes to skip before reading the STL file. - * @return Mesh - * @throws IOException If an error occurs during reading. - */ - private Mesh readBinary(InputStream is, int skip) throws IOException { - /* Prepare a ByteBuffer to read the rest of the STL file. */ - byte[] bytes = new byte[50]; - ByteBuffer buffer = ByteBuffer.wrap(bytes); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - /* Skip the STL header! */ - is.skip(skip); - - /* Read the bytes for the size (unsigned 32 bit int, little-endian). */ - byte[] sizeBytes = new byte[4]; - is.read(sizeBytes, 0, 4); - long triangles = ((sizeBytes[0] & 0xFF)) | ((sizeBytes[1] & 0xFF) << 8) | ((sizeBytes[2] & 0xFF) << 16) | ((sizeBytes[3] & 0xFF) << 24); - - /* TODO: Properly handle models whose triangles > MAX_TRIANGLES. */ - if (triangles <= 0) { - LOGGER.error("The number of triangles in the Mesh seems to be smaller than zero. This STL file is probably corrupt!"); - return null; - } else if (triangles > MAX_TRIANGLES) { - LOGGER.error("The number of triangles in the Mesh exceeds the limit that can currently be processed by STLMeshDecoder. The Mesh will be downsampled!"); - return null; - } - - /* Prepare Mesh. */ - Mesh mesh = new Mesh((int)triangles, (int)triangles); - - /* Prepare helper structures. */ - TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); - int index = 0; - int[] vertexindices = new int[3]; - - /* Now add all triangles. */ - for (int i=0; i MAX_TRIANGLES. */ + if (triangles <= 0) { + LOGGER.error("The number of triangles in the Mesh seems to be smaller than zero. This STL file is probably corrupt!"); + return null; + } else if (triangles > MAX_TRIANGLES) { + LOGGER.error("The number of triangles in the Mesh exceeds the limit that can currently be processed by STLMeshDecoder. The Mesh will be downsampled!"); + return null; } - /** - * Closes the Decoder. This method should cleanup and relinquish all resources. - *

    - * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - public void close() {} - - /** - * Indicates whether or not a particular instance of the Decoder interface can - * be re-used or not. This property can be leveraged to reduce the memory-footpring - * of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; - } + /* Prepare Mesh. */ + Mesh mesh = new Mesh((int) triangles, (int) triangles); + + /* Prepare helper structures. */ + TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); + int index = 0; + int[] vertexindices = new int[3]; + + /* Now add all triangles. */ + for (int i = 0; i < triangles; i++) { + /* Ready 48 bytes from the stream. */ + buffer.rewind(); + is.read(bytes); + + /* Read and ignore three floats. */ + buffer.getFloat(); + buffer.getFloat(); + buffer.getFloat(); + + /* Add the vertices and the vertex-normal to the mesh. */ + for (int vidx = 0; vidx < 3; vidx++) { + Vector3f vertex = new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat()); + if (!vertexBuffer.containsKey(vertex)) { + mesh.addVertex(vertex); + vertexBuffer.put(vertex, index); + index++; + } + vertexindices[vidx] = vertexBuffer.get(vertex); + } - /** - * Indicates whether or not the current decoder instance is complete i.e. if there is - * content left that can be obtained. - * - * @return true if there is still content, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + /* Add a new face to the Mesh. */ + if (!mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2]))) { + LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.", vertexindices[0], vertexindices[1], vertexindices[2]); + } } - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; - } + /* Closes the InputStream. */ + is.close(); + return mesh; + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; + } + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

    + * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/ShotBoundaryDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/ShotBoundaryDecoder.java index 91120ad9f..ab3b421c6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/ShotBoundaryDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/ShotBoundaryDecoder.java @@ -1,51 +1,51 @@ package org.vitrivr.cineast.core.extraction.decode.shotboundary; -import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; - import java.io.File; import java.io.FileNotFoundException; import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; public final class ShotBoundaryDecoder { - private ShotBoundaryDecoder(){} - - public static List decode(File boundaryFile, String videoId) throws NullPointerException, SecurityException, FileNotFoundException{ - if(boundaryFile == null){ - throw new NullPointerException("boundaryFile was null in ShotBoundaryDecoder.decode()"); - } - - if(videoId == null){ - throw new NullPointerException("videoId was null in ShotBoundaryDecoder.decode()"); - } - - if(!boundaryFile.exists() || !boundaryFile.isFile()){ - throw new FileNotFoundException("'" + boundaryFile.getAbsolutePath() + "' was not found in ShotBoundaryDecoder.decode()"); - } - - if(!boundaryFile.canRead()){ - throw new SecurityException("'" + boundaryFile.getAbsolutePath() + "' cannot be read in ShotBoundaryDecoder.decode()"); - } - - String extension = ""; - - int i = boundaryFile.getName().lastIndexOf('.'); - if (i > 0) { - extension = boundaryFile.getName().substring(i+1); - } - - switch(extension){ - - case "msb": - case "sb":{ - return TrecvidMasterShotReferenceDecoder.decode(boundaryFile, videoId); - } - - default: - throw new IllegalArgumentException("'" + extension + "' is not a recognized file extension in ShotBoundaryDecoder.decode()"); - } - - - } - + private ShotBoundaryDecoder() { + } + + public static List decode(File boundaryFile, String videoId) throws NullPointerException, SecurityException, FileNotFoundException { + if (boundaryFile == null) { + throw new NullPointerException("boundaryFile was null in ShotBoundaryDecoder.decode()"); + } + + if (videoId == null) { + throw new NullPointerException("videoId was null in ShotBoundaryDecoder.decode()"); + } + + if (!boundaryFile.exists() || !boundaryFile.isFile()) { + throw new FileNotFoundException("'" + boundaryFile.getAbsolutePath() + "' was not found in ShotBoundaryDecoder.decode()"); + } + + if (!boundaryFile.canRead()) { + throw new SecurityException("'" + boundaryFile.getAbsolutePath() + "' cannot be read in ShotBoundaryDecoder.decode()"); + } + + String extension = ""; + + int i = boundaryFile.getName().lastIndexOf('.'); + if (i > 0) { + extension = boundaryFile.getName().substring(i + 1); + } + + switch (extension) { + + case "msb": + case "sb": { + return TrecvidMasterShotReferenceDecoder.decode(boundaryFile, videoId); + } + + default: + throw new IllegalArgumentException("'" + extension + "' is not a recognized file extension in ShotBoundaryDecoder.decode()"); + } + + + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/TrecvidMasterShotReferenceDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/TrecvidMasterShotReferenceDecoder.java index 58c2b1774..524dcd546 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/TrecvidMasterShotReferenceDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/shotboundary/TrecvidMasterShotReferenceDecoder.java @@ -1,73 +1,78 @@ package org.vitrivr.cineast.core.extraction.decode.shotboundary; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; -import java.io.*; -import java.util.ArrayList; -import java.util.List; - public class TrecvidMasterShotReferenceDecoder { - private TrecvidMasterShotReferenceDecoder(){} - - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * decodes shot boundaries in the format used for TRECVID - * @param msr the file containing the master shot reference - * @param videoId the video id - */ - public static List decode(File msr, String videoId){ - - ArrayList _return = new ArrayList<>(); - - try { - BufferedReader reader = new BufferedReader(new FileReader(msr)); - String line = null; - int shotCounter = 0; - - while((line = reader.readLine()) != null){ - line = line.trim(); - - if(line.isEmpty()){ //skip empty lines - continue; - } - - if(!Character.isDigit(line.charAt(0))){//line does not start with a number - continue; - } - - String[] split = line.split(" "); - if(split.length < 2){//there are not two blocks on this line - continue; - } - - int start, end; - try{ - start = 1 + Integer.parseInt(split[0]); //TRECVID msr starts with 0 - end = 1 + Integer.parseInt(split[1]); - }catch(NumberFormatException e){ - continue; - } - - ++shotCounter; - - /* TODO: Derive absolute start and end position of MediaSegmentDescriptor. */ - _return.add(new MediaSegmentDescriptor(videoId, MediaType.generateSegmentId(MediaType.VIDEO, videoId, shotCounter), shotCounter, start, end, -1.0f, -1.0f, true)); - } - - reader.close(); - - } catch (FileNotFoundException e) { - LOGGER.error("error in TrecvidMasterShotReferenceDecoder.decode, file '{}' was not found", msr.getAbsolutePath()); - } catch (IOException e) { - LOGGER.error("error while reading file '{}' in TrecvidMasterShotReferenceDecoder.decode: {}", msr.getAbsolutePath(), e.getMessage()); - } - - return _return; - } - + private TrecvidMasterShotReferenceDecoder() { + } + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * decodes shot boundaries in the format used for TRECVID + * + * @param msr the file containing the master shot reference + * @param videoId the video id + */ + public static List decode(File msr, String videoId) { + + ArrayList _return = new ArrayList<>(); + + try { + BufferedReader reader = new BufferedReader(new FileReader(msr)); + String line = null; + int shotCounter = 0; + + while ((line = reader.readLine()) != null) { + line = line.trim(); + + if (line.isEmpty()) { //skip empty lines + continue; + } + + if (!Character.isDigit(line.charAt(0))) {//line does not start with a number + continue; + } + + String[] split = line.split(" "); + if (split.length < 2) {//there are not two blocks on this line + continue; + } + + int start, end; + try { + start = 1 + Integer.parseInt(split[0]); //TRECVID msr starts with 0 + end = 1 + Integer.parseInt(split[1]); + } catch (NumberFormatException e) { + continue; + } + + ++shotCounter; + + /* TODO: Derive absolute start and end position of MediaSegmentDescriptor. */ + _return.add(new MediaSegmentDescriptor(videoId, MediaType.generateSegmentId(MediaType.VIDEO, videoId, shotCounter), shotCounter, start, end, -1.0f, -1.0f, true)); + } + + reader.close(); + + } catch (FileNotFoundException e) { + LOGGER.error("error in TrecvidMasterShotReferenceDecoder.decode, file '{}' was not found", msr.getAbsolutePath()); + } catch (IOException e) { + LOGGER.error("error while reading file '{}' in TrecvidMasterShotReferenceDecoder.decode: {}", msr.getAbsolutePath(), e.getMessage()); + } + + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/AbstractSubtitleItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/AbstractSubtitleItem.java index f31e3ca57..59d941751 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/AbstractSubtitleItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/AbstractSubtitleItem.java @@ -1,42 +1,41 @@ package org.vitrivr.cineast.core.extraction.decode.subtitle; - public abstract class AbstractSubtitleItem implements SubtitleItem { - protected final int id; - protected final long start, end; - protected final String text; - - protected AbstractSubtitleItem(int id, long start, long end, String text){ - this.id = id; - this.start = start; - this.end = end; - this.text = text; - } - - /* (non-Javadoc) - * @see subsync.SubItem#getLength() - */ - @Override - public int getLength(){ - return (int) (end - start); - } - - - @Override - public long getStartTimestamp() { - return this.start; - } - - @Override - public long getEndTimestamp() { - return this.end; - } - - @Override - public String toString() { - return "id: " + id + "\n" + start + " ---> " + end + "\n" + text; - } + protected final int id; + protected final long start, end; + protected final String text; + + protected AbstractSubtitleItem(int id, long start, long end, String text) { + this.id = id; + this.start = start; + this.end = end; + this.text = text; + } + + /* (non-Javadoc) + * @see subsync.SubItem#getLength() + */ + @Override + public int getLength() { + return (int) (end - start); + } + + + @Override + public long getStartTimestamp() { + return this.start; + } + + @Override + public long getEndTimestamp() { + return this.end; + } + + @Override + public String toString() { + return "id: " + id + "\n" + start + " ---> " + end + "\n" + text; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubTitleDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubTitleDecoder.java index c82ac046b..014dee07f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubTitleDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubTitleDecoder.java @@ -4,47 +4,46 @@ import org.vitrivr.cineast.core.extraction.decode.video.FFMpegVideoDecoder; /** - * Represents a subtitle decoder instance. Basically, a {@link SubTitleDecoder} instance provides access to a stream of - * {@link SubtitleItem}s in temporal order. - * - * The decoder uses an internal pointer that can be incremented sequentially to access the latest items. Muxing with a stream of - * {@link VideoFrame}s is typically done via the presentation timestamp of both the {@link VideoFrame} and {@link SubtitleItem} + * Represents a subtitle decoder instance. Basically, a {@link SubTitleDecoder} instance provides access to a stream of {@link SubtitleItem}s in temporal order. + *

    + * The decoder uses an internal pointer that can be incremented sequentially to access the latest items. Muxing with a stream of {@link VideoFrame}s is typically done via the presentation timestamp of both the {@link VideoFrame} and {@link SubtitleItem} * * @see VideoFrame * @see FFMpegVideoDecoder */ public interface SubTitleDecoder { - /** - * Returns the number of {@link SubtitleItem}s in this {@link SubTitleDecoder}. - * - * @return Number of {@link SubtitleItem}s in this {@link SubTitleDecoder - */ - public abstract int getNumerOfItems(); - /** - * Returns the subtitle item at the specified index position. - * - * @param index Position index. - * @return {@link SubtitleItem} at the specified index position. - */ - public abstract SubtitleItem get(int index); + /** + * Returns the number of {@link SubtitleItem}s in this {@link SubTitleDecoder}. + * + * @return Number of {@link SubtitleItem}s in this {@link SubTitleDecoder + */ + public abstract int getNumerOfItems(); + + /** + * Returns the subtitle item at the specified index position. + * + * @param index Position index. + * @return {@link SubtitleItem} at the specified index position. + */ + public abstract SubtitleItem get(int index); - /** - * Returns the {@link SubtitleItem} at the current pointer position. - * - * @return {@link SubtitleItem} at the current pointer position. - */ - public abstract SubtitleItem getLast(); + /** + * Returns the {@link SubtitleItem} at the current pointer position. + * + * @return {@link SubtitleItem} at the current pointer position. + */ + public abstract SubtitleItem getLast(); - /** - * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. - * - * @return True if increment was successful and false otherwise. - */ - public abstract boolean increment(); + /** + * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. + * + * @return True if increment was successful and false otherwise. + */ + public abstract boolean increment(); - /** - * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. - */ - public abstract void rewind(); + /** + * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. + */ + public abstract void rewind(); } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleDecoderFactory.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleDecoderFactory.java index 956091c35..6266d351a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleDecoderFactory.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleDecoderFactory.java @@ -1,96 +1,93 @@ package org.vitrivr.cineast.core.extraction.decode.subtitle; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.extraction.decode.subtitle.cc.CCSubTitleDecoder; -import org.vitrivr.cineast.core.extraction.decode.subtitle.srt.SRTSubTitleDecoder; - import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.extraction.decode.subtitle.cc.CCSubTitleDecoder; +import org.vitrivr.cineast.core.extraction.decode.subtitle.srt.SRTSubTitleDecoder; /** - * This class can be used to instantiate {@link SubTitleDecoder} instances for a given video file or subtitle file. It - * uses the Java reflection API and a map of suffices to decoder classes to facilitate this functionality. + * This class can be used to instantiate {@link SubTitleDecoder} instances for a given video file or subtitle file. It uses the Java reflection API and a map of suffices to decoder classes to facilitate this functionality. * * Important: Classes that are supposed to be instantiated through this factory must have a public, * unary constructor with a {@link Path} argument. Failing to provide such a constructor will result in an runtime error. */ public final class SubtitleDecoderFactory { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); - /** - * Private constructor for this class cannot be instantiated. - */ - private SubtitleDecoderFactory() { } + /** + * Private constructor for this class cannot be instantiated. + */ + private SubtitleDecoderFactory() { + } - /** - * Map of the different subtitle file suffices and the associated {@link SubTitleDecoder} decoder instances. - */ - private static final Map> SUBTITLE_EXTENSION_MAP = new HashMap<>(); - static { - registerSubtitleDecoder(".srt", SRTSubTitleDecoder.class); - registerSubtitleDecoder(".cc", CCSubTitleDecoder.class); - } + /** + * Map of the different subtitle file suffices and the associated {@link SubTitleDecoder} decoder instances. + */ + private static final Map> SUBTITLE_EXTENSION_MAP = new HashMap<>(); - /** - * Tries to return a {@link SubTitleDecoder} object for the specified video file. This method expects a separate file with - * the same name as the video located in the same folder. If there is a registered decoder for the file, that decoer - * is instantiated and returned. - * - * @see SubtitleDecoderFactory#subtitleForFile - * - * @param path Path to the video file for which subtitles should be decoded. - * @return Optional {@link SubTitleDecoder} object. - */ - public static Optional subtitleForVideo(Path path) { - final String filename = path.getFileName().toString(); - final String suffix = filename.substring(filename.lastIndexOf('.'), filename.length()); - for (Map.Entry> entry : SUBTITLE_EXTENSION_MAP.entrySet()) { - final Path subtitleFile = path.getParent().resolve(filename.replace(suffix, entry.getKey())); - final Optional item = subtitleForFile(subtitleFile, entry.getValue()); - if (item.isPresent()) { - return item; - } - } + static { + registerSubtitleDecoder(".srt", SRTSubTitleDecoder.class); + registerSubtitleDecoder(".cc", CCSubTitleDecoder.class); + } - /* Returns an empty optional. */ - return Optional.empty(); + /** + * Tries to return a {@link SubTitleDecoder} object for the specified video file. This method expects a separate file with the same name as the video located in the same folder. If there is a registered decoder for the file, that decoer is instantiated and returned. + * + * @param path Path to the video file for which subtitles should be decoded. + * @return Optional {@link SubTitleDecoder} object. + * @see SubtitleDecoderFactory#subtitleForFile + */ + public static Optional subtitleForVideo(Path path) { + final String filename = path.getFileName().toString(); + final String suffix = filename.substring(filename.lastIndexOf('.'), filename.length()); + for (Map.Entry> entry : SUBTITLE_EXTENSION_MAP.entrySet()) { + final Path subtitleFile = path.getParent().resolve(filename.replace(suffix, entry.getKey())); + final Optional item = subtitleForFile(subtitleFile, entry.getValue()); + if (item.isPresent()) { + return item; + } } - /** - * Tries to decode the subtitle located under the specified path using the specified {@link SubTitleDecoder} decoder. If - * this method succeeds, a new {@link SubTitleDecoder} object is returned. Otherwise, the returned optional will be empty. - * - * @param path The subtitle file that should be decoded. - * @param clazz The {@link SubTitleDecoder} decoder that should be used to decode the file. - * @return Optional {@link SubTitleDecoder} object. - */ - public static Optional subtitleForFile(Path path, Class clazz) { - if (Files.isReadable(path)) { - try { - return Optional.of(clazz.getDeclaredConstructor(Path.class).newInstance(path)); - } catch (InstantiationException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { - LOGGER.error("Failed to instantiate the SubTitleDecoder decoder '{}'.", clazz.getName(), e); - } - } else { - LOGGER.warn("The specified file is not readable '{}'. Does it exist?", path); - } + /* Returns an empty optional. */ + return Optional.empty(); + } - /* Returns an empty optional. */ - return Optional.empty(); + /** + * Tries to decode the subtitle located under the specified path using the specified {@link SubTitleDecoder} decoder. If this method succeeds, a new {@link SubTitleDecoder} object is returned. Otherwise, the returned optional will be empty. + * + * @param path The subtitle file that should be decoded. + * @param clazz The {@link SubTitleDecoder} decoder that should be used to decode the file. + * @return Optional {@link SubTitleDecoder} object. + */ + public static Optional subtitleForFile(Path path, Class clazz) { + if (Files.isReadable(path)) { + try { + return Optional.of(clazz.getDeclaredConstructor(Path.class).newInstance(path)); + } catch (InstantiationException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { + LOGGER.error("Failed to instantiate the SubTitleDecoder decoder '{}'.", clazz.getName(), e); + } + } else { + LOGGER.warn("The specified file is not readable '{}'. Does it exist?", path); } - /** - * Registers a new {@link SubTitleDecoder} for a specific suffix. This method cannot be used to override already - * registered suffices! - */ - public static void registerSubtitleDecoder(String suffix, Class clazz) { - SUBTITLE_EXTENSION_MAP.putIfAbsent(suffix, clazz); - } + /* Returns an empty optional. */ + return Optional.empty(); + } + + /** + * Registers a new {@link SubTitleDecoder} for a specific suffix. This method cannot be used to override already registered suffices! + */ + public static void registerSubtitleDecoder(String suffix, Class clazz) { + SUBTITLE_EXTENSION_MAP.putIfAbsent(suffix, clazz); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleItem.java index a2b5926f2..dc3f946d4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/SubtitleItem.java @@ -1,36 +1,35 @@ package org.vitrivr.cineast.core.extraction.decode.subtitle; /** - * Represents a single subtitle item as found in videos. Typically, a subtitle item has a text, and a specified - * presentation time within the video. + * Represents a single subtitle item as found in videos. Typically, a subtitle item has a text, and a specified presentation time within the video. */ public interface SubtitleItem { - /** - * Returns the presentation length of the {@link SubtitleItem} in ms. - * - * @return presentation length of the {@link SubtitleItem} in ms - */ - int getLength(); + /** + * Returns the presentation length of the {@link SubtitleItem} in ms. + * + * @return presentation length of the {@link SubtitleItem} in ms + */ + int getLength(); - /** - * Returns the text of the {@link SubtitleItem}. - * - * @return Text of the {@link SubtitleItem}. - */ - String getText(); + /** + * Returns the text of the {@link SubtitleItem}. + * + * @return Text of the {@link SubtitleItem}. + */ + String getText(); - /** - * Returns the start presentation timestamp in ms. - * - * @return Start presentation timestamp in m - */ - long getStartTimestamp(); + /** + * Returns the start presentation timestamp in ms. + * + * @return Start presentation timestamp in m + */ + long getStartTimestamp(); - /** - * Returns the start presentation timestamp in ms. - * - * @return Start presentation timestamp in m - */ - long getEndTimestamp(); + /** + * Returns the start presentation timestamp in ms. + * + * @return Start presentation timestamp in m + */ + long getEndTimestamp(); } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java index 0ab34415d..30e0917d6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleDecoder.java @@ -1,11 +1,5 @@ package org.vitrivr.cineast.core.extraction.decode.subtitle.cc; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.extraction.decode.subtitle.SubTitleDecoder; -import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; -import org.vitrivr.cineast.core.util.LogHelper; - import java.io.BufferedReader; import java.io.IOException; import java.nio.file.Files; @@ -14,143 +8,148 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.extraction.decode.subtitle.SubTitleDecoder; +import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; +import org.vitrivr.cineast.core.util.LogHelper; public class CCSubTitleDecoder implements SubTitleDecoder { - private static final Logger LOGGER = LogManager.getLogger(); - private static final DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - - private int maxId = -1; - private long startTime = -1, endTime = -1; - private int pointer = 0; - private ArrayList items = new ArrayList<>(); - - public CCSubTitleDecoder(Path file){ - - LOGGER.info("Loading CC subtitle from {}", file); - - try (final BufferedReader reader = Files.newBufferedReader(file)) { - String line; - while((line = reader.readLine()) != null){ - String command = line.substring(0, 3); - switch (command) { - case "TOP":{ - String[] split = line.split("\\|"); - this.startTime = parseTime(split[1]); - break; - } - case "COL":{//ignore - break; - } - case "UID":{//ignore - break; - } - case "DUR":{//ignore - break; - } - case "VID":{//ignore - break; - } - case "SRC":{//ignore - break; - } - case "CMT":{//ignore - break; - } - case "LBT":{//ignore - break; - } - case "END":{ - String[] split = line.split("\\|"); - this.endTime = parseTime(split[1]); - break; - } - default:{ - String[] split = line.split("\\|"); - if(!split[2].startsWith("CC")){ - break; - } - - long start = parseFrame(split[0]); - long end = parseFrame(split[1]); - - int id = ++maxId; - - this.items.add(new CCSubTitleItem(id, start, end, split[3])); - - break; - } - } - - } - reader.close(); - - } catch (IOException e) { - LOGGER.warn("Error while loading subtitle"); - LOGGER.warn(LogHelper.getStackTrace(e)); - } - } - - private static long parseTime(String str){ - try{ - String s = str.substring(0,14); - return dateFormat.parse(s).getTime(); - }catch(ParseException e){ - LOGGER.error("error while parsing time {} in CCSubTitleDecoder", str); - return -1; - } - - } - - private long parseFrame(String str){ - - long dt = parseTime(str) - this.startTime; - - if(str.length() > 15){ - float frac = Float.parseFloat(str.substring(14)); - dt += 1000 * frac; - } - - return dt; - } - - @Override - public int getNumerOfItems() { - return this.items.size(); - } - - public SubtitleItem get(int index) { - return this.items.get(index); - } - - public SubtitleItem getLast() { - return this.items.get(pointer); - } - - /** - * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. - * - * @return True if increment was successful and false otherwise. - */ - public boolean increment() { - if (this.pointer + 1 < this.items.size()) { - this.pointer += 1; - return true; - } else { - return false; - } - } - - /** - * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. - */ - public void rewind() { - this.pointer = 0; - } - - @Override - public String toString() { - return "CC Subtitle, " + getNumerOfItems() + " elements, maxId: " + this.maxId + ", startTime: " + this.startTime + ", endTime: " + this.endTime; - } + private static final Logger LOGGER = LogManager.getLogger(); + private static final DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); + + private int maxId = -1; + private long startTime = -1, endTime = -1; + private int pointer = 0; + private ArrayList items = new ArrayList<>(); + + public CCSubTitleDecoder(Path file) { + + LOGGER.info("Loading CC subtitle from {}", file); + + try (final BufferedReader reader = Files.newBufferedReader(file)) { + String line; + while ((line = reader.readLine()) != null) { + String command = line.substring(0, 3); + switch (command) { + case "TOP": { + String[] split = line.split("\\|"); + this.startTime = parseTime(split[1]); + break; + } + case "COL": {//ignore + break; + } + case "UID": {//ignore + break; + } + case "DUR": {//ignore + break; + } + case "VID": {//ignore + break; + } + case "SRC": {//ignore + break; + } + case "CMT": {//ignore + break; + } + case "LBT": {//ignore + break; + } + case "END": { + String[] split = line.split("\\|"); + this.endTime = parseTime(split[1]); + break; + } + default: { + String[] split = line.split("\\|"); + if (!split[2].startsWith("CC")) { + break; + } + + long start = parseFrame(split[0]); + long end = parseFrame(split[1]); + + int id = ++maxId; + + this.items.add(new CCSubTitleItem(id, start, end, split[3])); + + break; + } + } + + } + reader.close(); + + } catch (IOException e) { + LOGGER.warn("Error while loading subtitle"); + LOGGER.warn(LogHelper.getStackTrace(e)); + } + } + + private static long parseTime(String str) { + try { + String s = str.substring(0, 14); + return dateFormat.parse(s).getTime(); + } catch (ParseException e) { + LOGGER.error("error while parsing time {} in CCSubTitleDecoder", str); + return -1; + } + + } + + private long parseFrame(String str) { + + long dt = parseTime(str) - this.startTime; + + if (str.length() > 15) { + float frac = Float.parseFloat(str.substring(14)); + dt += 1000 * frac; + } + + return dt; + } + + @Override + public int getNumerOfItems() { + return this.items.size(); + } + + public SubtitleItem get(int index) { + return this.items.get(index); + } + + public SubtitleItem getLast() { + return this.items.get(pointer); + } + + /** + * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. + * + * @return True if increment was successful and false otherwise. + */ + public boolean increment() { + if (this.pointer + 1 < this.items.size()) { + this.pointer += 1; + return true; + } else { + return false; + } + } + + /** + * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. + */ + public void rewind() { + this.pointer = 0; + } + + @Override + public String toString() { + return "CC Subtitle, " + getNumerOfItems() + " elements, maxId: " + this.maxId + ", startTime: " + this.startTime + ", endTime: " + this.endTime; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleItem.java index 6ddbfb95e..65e1bab45 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/cc/CCSubTitleItem.java @@ -4,12 +4,12 @@ public class CCSubTitleItem extends AbstractSubtitleItem { - protected CCSubTitleItem(int id, long start, long end, String text) { - super(id, start, end, text); - } - - @Override - public String getText() { - return this.text.trim(); - } + protected CCSubTitleItem(int id, long start, long end, String text) { + super(id, start, end, text); + } + + @Override + public String getText() { + return this.text.trim(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubTitleDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubTitleDecoder.java index a429cd9db..afb5c0acf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubTitleDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubTitleDecoder.java @@ -1,161 +1,160 @@ package org.vitrivr.cineast.core.extraction.decode.subtitle.srt; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.extraction.decode.subtitle.SubTitleDecoder; -import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; -import org.vitrivr.cineast.core.util.LogHelper; - import java.io.BufferedReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.extraction.decode.subtitle.SubTitleDecoder; +import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; +import org.vitrivr.cineast.core.util.LogHelper; public class SRTSubTitleDecoder implements SubTitleDecoder { - private static final Logger LOGGER = LogManager.getLogger(); - - private List items = new ArrayList<>(); - - private int maxId = -1; - private long startTime = -1, endTime = -1; - private int pointer = 0; - - public SRTSubTitleDecoder(Path file) { - LOGGER.info("Loading SRT subtitle from {}", file); - try (final BufferedReader reader = Files.newBufferedReader(file)) { - String line1, line2, line3; - StringBuffer text; - int id = 0; - long start, end; - loop: - while ((line1 = reader.readLine()) != null) { - - while (line1.isEmpty()) { - line1 = reader.readLine(); - if (line1 == null) { - break loop; - } - } - - try { - id = Integer.parseInt(line1); - - this.maxId = Math.max(maxId, id); - - line2 = reader.readLine(); - if (line2 == null) { - break; - } - String[] timing = line2.split(" --> "); - if (timing.length != 2) { - break; - } - - start = parseTime(timing[0]); - end = parseTime(timing[1]); - - if (this.startTime == -1) { - this.startTime = start; - } - - this.endTime = end; - - text = new StringBuffer(); - while ((line3 = reader.readLine()) != null && !line3.isEmpty()) { - text.append(line3); - text.append('\n'); - } - - items.add(new SRTSubtitleItem(id, start, end, text.toString())); - } catch (NumberFormatException e) { - LOGGER.warn("Error while parsing subtitle item"); - LOGGER.warn(LogHelper.getStackTrace(e)); - } - } - - reader.close(); - } catch (IOException e) { - LOGGER.warn("Error while loading subtitle"); - LOGGER.warn(LogHelper.getStackTrace(e)); + private static final Logger LOGGER = LogManager.getLogger(); + + private List items = new ArrayList<>(); + + private int maxId = -1; + private long startTime = -1, endTime = -1; + private int pointer = 0; + + public SRTSubTitleDecoder(Path file) { + LOGGER.info("Loading SRT subtitle from {}", file); + try (final BufferedReader reader = Files.newBufferedReader(file)) { + String line1, line2, line3; + StringBuffer text; + int id = 0; + long start, end; + loop: + while ((line1 = reader.readLine()) != null) { + + while (line1.isEmpty()) { + line1 = reader.readLine(); + if (line1 == null) { + break loop; + } } - } - private static long parseTime(String time) { - long h = 0, m = 0, s = 0, ms = 0; - String[] splits = time.split(":"); - if (splits.length != 3) { - return -1; - } + try { + id = Integer.parseInt(line1); - h = Long.parseLong(splits[0]); - m = Long.parseLong(splits[1]); - splits = splits[2].split(","); + this.maxId = Math.max(maxId, id); - if (splits.length != 2) { - return -1; - } + line2 = reader.readLine(); + if (line2 == null) { + break; + } + String[] timing = line2.split(" --> "); + if (timing.length != 2) { + break; + } - s = Long.parseLong(splits[0]); - ms = Long.parseLong(splits[1]); + start = parseTime(timing[0]); + end = parseTime(timing[1]); - return ms + 1000L * s + 60000L * m + 3600000L * h; - } + if (this.startTime == -1) { + this.startTime = start; + } - /** - * Returns the number of {@link SubtitleItem}s in this {@link SubTitleDecoder}. - * - * @return Number of {@link SubtitleItem}s in this {@link SubTitleDecoder - */ - @Override - public int getNumerOfItems() { - return this.items.size(); - } + this.endTime = end; - /** - * Returns the subtitle item at the specified index position. - * - * @param index Position index. - * @return {@link SubtitleItem} at the specified index position. - */ - public SubtitleItem get(int index) { - return this.items.get(index); - } + text = new StringBuffer(); + while ((line3 = reader.readLine()) != null && !line3.isEmpty()) { + text.append(line3); + text.append('\n'); + } + + items.add(new SRTSubtitleItem(id, start, end, text.toString())); + } catch (NumberFormatException e) { + LOGGER.warn("Error while parsing subtitle item"); + LOGGER.warn(LogHelper.getStackTrace(e)); + } + } - /** - * Returns the {@link SubtitleItem} at the current pointer position. - * - * @return {@link SubtitleItem} at the current pointer position. - */ - public SubtitleItem getLast() { - return this.items.get(pointer); + reader.close(); + } catch (IOException e) { + LOGGER.warn("Error while loading subtitle"); + LOGGER.warn(LogHelper.getStackTrace(e)); } + } - /** - * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. - * - * @return True if increment was successful and false otherwise. - */ - public boolean increment() { - if (this.pointer + 1 < this.items.size()) { - this.pointer += 1; - return true; - } else { - return false; - } + private static long parseTime(String time) { + long h = 0, m = 0, s = 0, ms = 0; + String[] splits = time.split(":"); + if (splits.length != 3) { + return -1; } - /** - * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. - */ - public void rewind() { - this.pointer = 0; + h = Long.parseLong(splits[0]); + m = Long.parseLong(splits[1]); + splits = splits[2].split(","); + + if (splits.length != 2) { + return -1; } - @Override - public String toString() { - return "SRT Subtitle, " + getNumerOfItems() + " elements, maxId: " + this.maxId + ", startTime: " + this.startTime + ", endTime: " + this.endTime; + s = Long.parseLong(splits[0]); + ms = Long.parseLong(splits[1]); + + return ms + 1000L * s + 60000L * m + 3600000L * h; + } + + /** + * Returns the number of {@link SubtitleItem}s in this {@link SubTitleDecoder}. + * + * @return Number of {@link SubtitleItem}s in this {@link SubTitleDecoder + */ + @Override + public int getNumerOfItems() { + return this.items.size(); + } + + /** + * Returns the subtitle item at the specified index position. + * + * @param index Position index. + * @return {@link SubtitleItem} at the specified index position. + */ + public SubtitleItem get(int index) { + return this.items.get(index); + } + + /** + * Returns the {@link SubtitleItem} at the current pointer position. + * + * @return {@link SubtitleItem} at the current pointer position. + */ + public SubtitleItem getLast() { + return this.items.get(pointer); + } + + /** + * Increments the internal pointer by one. Returns true, if increment was successful and false otherwise. + * + * @return True if increment was successful and false otherwise. + */ + public boolean increment() { + if (this.pointer + 1 < this.items.size()) { + this.pointer += 1; + return true; + } else { + return false; } + } + + /** + * Rewinds the {@link SubTitleDecoder} stream and sets the internal pointer to 0. + */ + public void rewind() { + this.pointer = 0; + } + + @Override + public String toString() { + return "SRT Subtitle, " + getNumerOfItems() + " elements, maxId: " + this.maxId + ", startTime: " + this.startTime + ", endTime: " + this.endTime; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubtitleItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubtitleItem.java index 0fc9a84b6..b0368305b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubtitleItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/subtitle/srt/SRTSubtitleItem.java @@ -4,20 +4,20 @@ public class SRTSubtitleItem extends AbstractSubtitleItem { - /** - * - */ - public SRTSubtitleItem(int id, long start, long end, String text){ - super(id, start, end, text); - } + /** + * + */ + public SRTSubtitleItem(int id, long start, long end, String text) { + super(id, start, end, text); + } + + /* (non-Javadoc) + * @see subsync.SubItem#getText() + */ + @Override + public String getText() { + return this.text.replaceAll("<[^>]*>", ""); + } - /* (non-Javadoc) - * @see subsync.SubItem#getText() - */ - @Override - public String getText(){ - return this.text.replaceAll("<[^>]*>", ""); - } - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AbstractAVStreamContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AbstractAVStreamContainer.java index d9b82c2e0..df767b40b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AbstractAVStreamContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AbstractAVStreamContainer.java @@ -8,66 +8,66 @@ public abstract class AbstractAVStreamContainer { - private static final Logger LOGGER = LogManager.getLogger(); - avcodec.AVPacket pkt; - protected avformat.AVStream st; - protected avcodec.AVCodec codec; - avcodec.AVCodecContext c; - private avformat.AVFormatContext oc; - - AbstractAVStreamContainer(avformat.AVFormatContext oc, int codec_id) { - this.oc = oc; - st = avformat.avformat_new_stream(oc, null); - if (st == null) { - LOGGER.error("Could not allocate stream"); - return; - } - st.id(oc.nb_streams() - 1); - - codec = avcodec.avcodec_find_encoder(codec_id); - if (codec == null) { - LOGGER.error("Could not find codec for id " + codec_id); - return; - } - - c = avcodec.avcodec_alloc_context3(codec); - if (c == null) { - LOGGER.error("Could not alloc an encoding context"); - return; - } - - c.codec_id(codec_id); - - pkt = avcodec.av_packet_alloc(); + private static final Logger LOGGER = LogManager.getLogger(); + avcodec.AVPacket pkt; + protected avformat.AVStream st; + protected avcodec.AVCodec codec; + avcodec.AVCodecContext c; + private avformat.AVFormatContext oc; + + AbstractAVStreamContainer(avformat.AVFormatContext oc, int codec_id) { + this.oc = oc; + st = avformat.avformat_new_stream(oc, null); + if (st == null) { + LOGGER.error("Could not allocate stream"); + return; } + st.id(oc.nb_streams() - 1); + codec = avcodec.avcodec_find_encoder(codec_id); + if (codec == null) { + LOGGER.error("Could not find codec for id " + codec_id); + return; + } + + c = avcodec.avcodec_alloc_context3(codec); + if (c == null) { + LOGGER.error("Could not alloc an encoding context"); + return; + } + + c.codec_id(codec_id); - void encode(avcodec.AVCodecContext enc_ctx, avutil.AVFrame frame, avcodec.AVPacket pkt) { + pkt = avcodec.av_packet_alloc(); + } - int ret = avcodec.avcodec_send_frame(enc_ctx, frame); - if (ret < 0) { - LOGGER.error("Error sending a frame for encoding"); - return; - } - while (ret >= 0) { - ret = avcodec.avcodec_receive_packet(enc_ctx, pkt); - if (ret == avutil.AVERROR_EAGAIN() - || - ret == avutil.AVERROR_EOF()) { - return; - } else if (ret < 0) { - LOGGER.error("Error during encoding"); - } + void encode(avcodec.AVCodecContext enc_ctx, avutil.AVFrame frame, avcodec.AVPacket pkt) { + + int ret = avcodec.avcodec_send_frame(enc_ctx, frame); + if (ret < 0) { + LOGGER.error("Error sending a frame for encoding"); + return; + } + + while (ret >= 0) { + ret = avcodec.avcodec_receive_packet(enc_ctx, pkt); + if (ret == avutil.AVERROR_EAGAIN() + || + ret == avutil.AVERROR_EOF()) { + return; + } else if (ret < 0) { + LOGGER.error("Error during encoding"); + } - avcodec.av_packet_rescale_ts(pkt, c.time_base(), st.time_base()); - pkt.stream_index(st.index()); + avcodec.av_packet_rescale_ts(pkt, c.time_base(), st.time_base()); + pkt.stream_index(st.index()); - /* Write the compressed frame to the media file. */ - avformat.av_interleaved_write_frame(oc, pkt); + /* Write the compressed frame to the media file. */ + avformat.av_interleaved_write_frame(oc, pkt); - avcodec.av_packet_unref(pkt); - } + avcodec.av_packet_unref(pkt); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AudioOutputStreamContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AudioOutputStreamContainer.java index d3b8cfe33..dddb5ed6b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AudioOutputStreamContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AudioOutputStreamContainer.java @@ -1,5 +1,26 @@ package org.vitrivr.cineast.core.extraction.decode.video; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_CAP_VARIABLE_FRAME_SIZE; +import static org.bytedeco.javacpp.avcodec.avcodec_free_context; +import static org.bytedeco.javacpp.avcodec.avcodec_parameters_from_context; +import static org.bytedeco.javacpp.avutil.AVDictionary; +import static org.bytedeco.javacpp.avutil.AVFrame; +import static org.bytedeco.javacpp.avutil.AVRational; +import static org.bytedeco.javacpp.avutil.AV_ROUND_UP; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_FLTP; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S16; +import static org.bytedeco.javacpp.avutil.av_frame_free; +import static org.bytedeco.javacpp.avutil.av_frame_make_writable; +import static org.bytedeco.javacpp.avutil.av_opt_set_int; +import static org.bytedeco.javacpp.avutil.av_opt_set_sample_fmt; +import static org.bytedeco.javacpp.avutil.av_rescale_q; +import static org.bytedeco.javacpp.avutil.av_rescale_rnd; +import static org.bytedeco.javacpp.swresample.swr_alloc; +import static org.bytedeco.javacpp.swresample.swr_convert; +import static org.bytedeco.javacpp.swresample.swr_free; +import static org.bytedeco.javacpp.swresample.swr_get_delay; +import static org.bytedeco.javacpp.swresample.swr_init; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bytedeco.javacpp.avcodec; @@ -8,210 +29,205 @@ import org.bytedeco.javacpp.swresample; import org.vitrivr.cineast.core.data.frames.AudioFrame; -import static org.bytedeco.javacpp.avcodec.*; -import static org.bytedeco.javacpp.avutil.*; -import static org.bytedeco.javacpp.swresample.*; - class AudioOutputStreamContainer extends AbstractAVStreamContainer { - private static final Logger LOGGER = LogManager.getLogger(); - long next_pts; - AVFrame tmp_frame; - private int samples_count; - private AVFrame frame; - private swresample.SwrContext swr_ctx; - private AVRational rat = new AVRational(); + private static final Logger LOGGER = LogManager.getLogger(); + long next_pts; + AVFrame tmp_frame; + private int samples_count; + private AVFrame frame; + private swresample.SwrContext swr_ctx; + private AVRational rat = new AVRational(); - private final int channels = 1; + private final int channels = 1; - AudioOutputStreamContainer(avformat.AVFormatContext oc, int codec_id, int sampleRate, int bitRate, AVDictionary opt) { - super(oc, codec_id); + AudioOutputStreamContainer(avformat.AVFormatContext oc, int codec_id, int sampleRate, int bitRate, AVDictionary opt) { + super(oc, codec_id); - long channellayout = avutil.av_get_default_channel_layout(channels); + long channellayout = avutil.av_get_default_channel_layout(channels); - if (codec.type() != avutil.AVMEDIA_TYPE_AUDIO) { - LOGGER.error("Not an audio codec"); - return; - } + if (codec.type() != avutil.AVMEDIA_TYPE_AUDIO) { + LOGGER.error("Not an audio codec"); + return; + } - c.sample_fmt(!codec.sample_fmts().isNull() ? - codec.sample_fmts().get(0) : AV_SAMPLE_FMT_FLTP); - c.bit_rate(bitRate); - c.sample_rate(sampleRate); - if (!codec.supported_samplerates().isNull()) { - c.sample_rate(codec.supported_samplerates().get(0)); - int i = 0; - int lastSampleRate; - while ((lastSampleRate = codec.supported_samplerates().get(++i)) > 0){ - if (lastSampleRate == sampleRate) { - c.sample_rate(sampleRate); - break; - } - } - } - - c.channel_layout(channellayout); - c.channels(avutil.av_get_channel_layout_nb_channels(c.channel_layout())); - - if (codec.channel_layouts() != null) { - c.channel_layout(codec.channel_layouts().get(0)); - int i = 0; - long lastChannelLayout; - while ((lastChannelLayout = codec.channel_layouts().get(++i)) > 0){ - if (lastChannelLayout == channellayout){ - c.channel_layout(channellayout); - break; - } - } - } - c.channels(avutil.av_get_channel_layout_nb_channels(c.channel_layout())); - AVRational timeBase = new AVRational(); - timeBase.num(1); - timeBase.den(c.sample_rate()); - st.time_base(timeBase); + c.sample_fmt(!codec.sample_fmts().isNull() ? + codec.sample_fmts().get(0) : AV_SAMPLE_FMT_FLTP); + c.bit_rate(bitRate); + c.sample_rate(sampleRate); + if (!codec.supported_samplerates().isNull()) { + c.sample_rate(codec.supported_samplerates().get(0)); + int i = 0; + int lastSampleRate; + while ((lastSampleRate = codec.supported_samplerates().get(++i)) > 0) { + if (lastSampleRate == sampleRate) { + c.sample_rate(sampleRate); + break; + } + } + } + c.channel_layout(channellayout); + c.channels(avutil.av_get_channel_layout_nb_channels(c.channel_layout())); - if ((oc.oformat().flags() & avformat.AVFMT_GLOBALHEADER) != 0) { - oc.oformat().flags(oc.oformat().flags() | avformat.AVFMT_GLOBALHEADER); + if (codec.channel_layouts() != null) { + c.channel_layout(codec.channel_layouts().get(0)); + int i = 0; + long lastChannelLayout; + while ((lastChannelLayout = codec.channel_layouts().get(++i)) > 0) { + if (lastChannelLayout == channellayout) { + c.channel_layout(channellayout); + break; } + } + } + c.channels(avutil.av_get_channel_layout_nb_channels(c.channel_layout())); + AVRational timeBase = new AVRational(); + timeBase.num(1); + timeBase.den(c.sample_rate()); + st.time_base(timeBase); + + if ((oc.oformat().flags() & avformat.AVFMT_GLOBALHEADER) != 0) { + oc.oformat().flags(oc.oformat().flags() | avformat.AVFMT_GLOBALHEADER); + } - int nb_samples; - - AVDictionary topt = new AVDictionary(); - - avutil.av_dict_copy(topt, opt, 0); - int ret = avcodec.avcodec_open2(c, codec, topt); - avutil.av_dict_free(topt); + int nb_samples; - if (ret < 0) { - LOGGER.error("Could not open audio codec: {}", ret); - return; - } + AVDictionary topt = new AVDictionary(); + avutil.av_dict_copy(topt, opt, 0); + int ret = avcodec.avcodec_open2(c, codec, topt); + avutil.av_dict_free(topt); - if ((c.codec().capabilities() & AV_CODEC_CAP_VARIABLE_FRAME_SIZE) != 0) - nb_samples = 10000; - else - nb_samples = c.frame_size(); + if (ret < 0) { + LOGGER.error("Could not open audio codec: {}", ret); + return; + } - frame = alloc_audio_frame(c.sample_fmt(), c.channel_layout(), c.sample_rate(), nb_samples); - tmp_frame = alloc_audio_frame(AV_SAMPLE_FMT_S16, channellayout, sampleRate, nb_samples); + if ((c.codec().capabilities() & AV_CODEC_CAP_VARIABLE_FRAME_SIZE) != 0) { + nb_samples = 10000; + } else { + nb_samples = c.frame_size(); + } - /* copy the stream parameters to the muxer */ - ret = avcodec_parameters_from_context(st.codecpar(), c); - if (ret < 0) { - LOGGER.error("Could not copy the stream parameters"); - return; - } + frame = alloc_audio_frame(c.sample_fmt(), c.channel_layout(), c.sample_rate(), nb_samples); + tmp_frame = alloc_audio_frame(AV_SAMPLE_FMT_S16, channellayout, sampleRate, nb_samples); - /* create resampler context */ - swr_ctx = swr_alloc(); - if (swr_ctx == null) { - LOGGER.error("Could not allocate resampler context"); - return; - } + /* copy the stream parameters to the muxer */ + ret = avcodec_parameters_from_context(st.codecpar(), c); + if (ret < 0) { + LOGGER.error("Could not copy the stream parameters"); + return; + } - /* set options */ - av_opt_set_int(swr_ctx, "in_channel_count", c.channels(), 0); - av_opt_set_int(swr_ctx, "in_sample_rate", c.sample_rate(), 0); //this should use input rather than output sampling rate, but doing so crashes the application... - av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); - av_opt_set_int(swr_ctx, "out_channel_count", c.channels(), 0); - av_opt_set_int(swr_ctx, "out_sample_rate", c.sample_rate(), 0); - av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", c.sample_fmt(), 0); - - /* initialize the resampling context */ - if ((swr_init(swr_ctx)) < 0) { - LOGGER.error("Failed to initialize the resampling context"); - } + /* create resampler context */ + swr_ctx = swr_alloc(); + if (swr_ctx == null) { + LOGGER.error("Could not allocate resampler context"); + return; + } + /* set options */ + av_opt_set_int(swr_ctx, "in_channel_count", c.channels(), 0); + av_opt_set_int(swr_ctx, "in_sample_rate", c.sample_rate(), 0); //this should use input rather than output sampling rate, but doing so crashes the application... + av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(swr_ctx, "out_channel_count", c.channels(), 0); + av_opt_set_int(swr_ctx, "out_sample_rate", c.sample_rate(), 0); + av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", c.sample_fmt(), 0); + + /* initialize the resampling context */ + if ((swr_init(swr_ctx)) < 0) { + LOGGER.error("Failed to initialize the resampling context"); } - private static AVFrame alloc_audio_frame(int sample_fmt, long channel_layout, int sample_rate, int nb_samples) { + } - AVFrame frame = avutil.av_frame_alloc(); + private static AVFrame alloc_audio_frame(int sample_fmt, long channel_layout, int sample_rate, int nb_samples) { - if (frame == null) { - LOGGER.error("Error allocating an audio frame"); - return null; - } + AVFrame frame = avutil.av_frame_alloc(); - frame.format(sample_fmt); - frame.channel_layout(channel_layout); - frame.sample_rate(sample_rate); - frame.nb_samples(nb_samples); - - if (nb_samples > 0) { - int ret = avutil.av_frame_get_buffer(frame, 0); - if (ret < 0) { - LOGGER.error("Error allocating an audio buffer"); - return null; - } - } + if (frame == null) { + LOGGER.error("Error allocating an audio frame"); + return null; + } - return frame; + frame.format(sample_fmt); + frame.channel_layout(channel_layout); + frame.sample_rate(sample_rate); + frame.nb_samples(nb_samples); + + if (nb_samples > 0) { + int ret = avutil.av_frame_get_buffer(frame, 0); + if (ret < 0) { + LOGGER.error("Error allocating an audio buffer"); + return null; + } } - void close() { - if (c != null && !c.isNull()) { - avcodec_free_context(c); - } - if (frame != null && !frame.isNull()) { - av_frame_free(frame); - } - if (tmp_frame != null && !tmp_frame.isNull()) { - av_frame_free(tmp_frame); - } - if (swr_ctx != null && !swr_ctx.isNull()) { - swr_free(swr_ctx); - } + return frame; + } + + void close() { + if (c != null && !c.isNull()) { + avcodec_free_context(c); } + if (frame != null && !frame.isNull()) { + av_frame_free(frame); + } + if (tmp_frame != null && !tmp_frame.isNull()) { + av_frame_free(tmp_frame); + } + if (swr_ctx != null && !swr_ctx.isNull()) { + swr_free(swr_ctx); + } + } - void addFrame(AudioFrame av) { - AVFrame inputFrame = tmp_frame; - int ret; - int dst_nb_samples; + void addFrame(AudioFrame av) { + AVFrame inputFrame = tmp_frame; + int ret; + int dst_nb_samples; - inputFrame.data().put(av.getData()); + inputFrame.data().put(av.getData()); - inputFrame.pts(next_pts); - next_pts += inputFrame.nb_samples(); + inputFrame.pts(next_pts); + next_pts += inputFrame.nb_samples(); - /* convert samples from native format to destination codec format, using the resampler */ - /* compute destination number of samples */ - dst_nb_samples = (int) av_rescale_rnd(swr_get_delay(swr_ctx, c.sample_rate()) + inputFrame.nb_samples(), - c.sample_rate(), c.sample_rate(), AV_ROUND_UP); + /* convert samples from native format to destination codec format, using the resampler */ + /* compute destination number of samples */ + dst_nb_samples = (int) av_rescale_rnd(swr_get_delay(swr_ctx, c.sample_rate()) + inputFrame.nb_samples(), + c.sample_rate(), c.sample_rate(), AV_ROUND_UP); - /* when we pass a frame to the encoder, it may keep a reference to it - * internally; - * make sure we do not overwrite it here - */ - ret = av_frame_make_writable(frame); - if (ret < 0) { - return; - } + /* when we pass a frame to the encoder, it may keep a reference to it + * internally; + * make sure we do not overwrite it here + */ + ret = av_frame_make_writable(frame); + if (ret < 0) { + return; + } - /* convert to destination format */ - ret = swr_convert(swr_ctx, - frame.data(), dst_nb_samples, - inputFrame.data(), inputFrame.nb_samples()); - if (ret < 0) { - LOGGER.error("Error while converting"); - return; - } + /* convert to destination format */ + ret = swr_convert(swr_ctx, + frame.data(), dst_nb_samples, + inputFrame.data(), inputFrame.nb_samples()); + if (ret < 0) { + LOGGER.error("Error while converting"); + return; + } - rat.num(1); - rat.den(c.sample_rate()); + rat.num(1); + rat.den(c.sample_rate()); - frame.pts(av_rescale_q(samples_count, rat, c.time_base())); - samples_count += dst_nb_samples; + frame.pts(av_rescale_q(samples_count, rat, c.time_base())); + samples_count += dst_nb_samples; - encode(c, frame, pkt); + encode(c, frame, pkt); - } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java index cc77011f3..cbd25f4a0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java @@ -1,669 +1,714 @@ package org.vitrivr.cineast.core.extraction.decode.video; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.avcodec; import org.bytedeco.javacpp.avcodec.AVCodec; +import org.bytedeco.javacpp.avformat; import org.bytedeco.javacpp.avformat.AVFormatContext; +import org.bytedeco.javacpp.avutil; import org.bytedeco.javacpp.avutil.AVDictionary; import org.bytedeco.javacpp.avutil.AVRational; -import org.vitrivr.cineast.core.config.DecoderConfig; +import org.bytedeco.javacpp.swresample; +import org.bytedeco.javacpp.swscale; import org.vitrivr.cineast.core.config.CacheConfig; -import org.vitrivr.cineast.core.data.raw.CachedDataFactory; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; +import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.data.frames.AudioDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.data.frames.VideoDescriptor; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.extraction.decode.subtitle.SubTitleDecoder; import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleDecoderFactory; import org.vitrivr.cineast.core.extraction.decode.subtitle.SubtitleItem; import org.vitrivr.cineast.core.util.LogHelper; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - /** * A {@link Decoder} implementation that decodes videos using the ffmpeg library + the corresponding Java bindings. - * */ public class FFMpegVideoDecoder implements Decoder { - /** Configuration property name for the {@link FFMpegVideoDecoder}: max width of the converted video. */ - private static final String CONFIG_MAXWIDTH_PROPERTY = "maxFrameWidth"; - - /** Configuration property name for the {@link FFMpegVideoDecoder}: max height of the converted video. */ - private static final String CONFIG_HEIGHT_PROPERTY = "maxFrameHeight"; - - /** Configuration property name for the {@link FFMpegVideoDecoder}: number of channels of the converted audio. If <= 0, then no audio will be decoded. */ - private static final String CONFIG_CHANNELS_PROPERTY = "channels"; - /** Configuration property name for the {@link FFMpegVideoDecoder}: samplerate of the converted audio. */ - private static final String CONFIG_SAMPLERATE_PROPERTY = "samplerate"; - - /** Configuration property name for the {@link FFMpegVideoDecoder}: Indicates whether subtitles should be decoded as well. */ - private static final String CONFIG_SUBTITLE_PROPERTY= "subtitles"; - - /** Configuration property default for the FFMpegVideoDecoder: max width of the converted video. */ - private final static int CONFIG_MAXWIDTH_DEFAULT = 1920; - - /** Configuration property default for the FFMpegVideoDecoder: max height of the converted video. */ - private final static int CONFIG_MAXHEIGHT_DEFAULT = 1080; - - /** Configuration property default for the FFMpegVideoDecoder: number of channels of the converted audio. */ - private static final int CONFIG_CHANNELS_DEFAULT = 1; - - /** Configuration property default for the FFMpegVideoDecoder: sample rate of the converted audio */ - private static final int CONFIG_SAMPLERATE_DEFAULT = 44100; - - private static final int TARGET_FORMAT = avutil.AV_SAMPLE_FMT_S16; - private static final int BYTES_PER_SAMPLE = avutil.av_get_bytes_per_sample(TARGET_FORMAT); - - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * Lists the mime types supported by the FFMpegVideoDecoder. - * - * TODO: List may not be complete yet. - */ - public static final Set supportedFiles; - static { - HashSet tmp = new HashSet<>(); - tmp.add("multimedia/mp4"); /* They share the same suffix with video (.mp4). */ - tmp.add("video/mp4"); - tmp.add("video/avi"); - tmp.add("video/mpeg"); - tmp.add("video/quicktime"); - tmp.add("video/webm"); - supportedFiles = Collections.unmodifiableSet(tmp); + /** + * Configuration property name for the {@link FFMpegVideoDecoder}: max width of the converted video. + */ + private static final String CONFIG_MAXWIDTH_PROPERTY = "maxFrameWidth"; + + /** + * Configuration property name for the {@link FFMpegVideoDecoder}: max height of the converted video. + */ + private static final String CONFIG_HEIGHT_PROPERTY = "maxFrameHeight"; + + /** + * Configuration property name for the {@link FFMpegVideoDecoder}: number of channels of the converted audio. If <= 0, then no audio will be decoded. + */ + private static final String CONFIG_CHANNELS_PROPERTY = "channels"; + + /** + * Configuration property name for the {@link FFMpegVideoDecoder}: samplerate of the converted audio. + */ + private static final String CONFIG_SAMPLERATE_PROPERTY = "samplerate"; + + /** + * Configuration property name for the {@link FFMpegVideoDecoder}: Indicates whether subtitles should be decoded as well. + */ + private static final String CONFIG_SUBTITLE_PROPERTY = "subtitles"; + + /** + * Configuration property default for the FFMpegVideoDecoder: max width of the converted video. + */ + private final static int CONFIG_MAXWIDTH_DEFAULT = 1920; + + /** + * Configuration property default for the FFMpegVideoDecoder: max height of the converted video. + */ + private final static int CONFIG_MAXHEIGHT_DEFAULT = 1080; + + /** + * Configuration property default for the FFMpegVideoDecoder: number of channels of the converted audio. + */ + private static final int CONFIG_CHANNELS_DEFAULT = 1; + + /** + * Configuration property default for the FFMpegVideoDecoder: sample rate of the converted audio + */ + private static final int CONFIG_SAMPLERATE_DEFAULT = 44100; + + private static final int TARGET_FORMAT = avutil.AV_SAMPLE_FMT_S16; + private static final int BYTES_PER_SAMPLE = avutil.av_get_bytes_per_sample(TARGET_FORMAT); + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Lists the mime types supported by the FFMpegVideoDecoder. + *

    + * TODO: List may not be complete yet. + */ + public static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("multimedia/mp4"); /* They share the same suffix with video (.mp4). */ + tmp.add("video/mp4"); + tmp.add("video/avi"); + tmp.add("video/mpeg"); + tmp.add("video/quicktime"); + tmp.add("video/webm"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + private byte[] bytes; + private int[] pixels; + + /** + * Internal data structure used to hold decoded VideoFrames and the associated timestamp. + */ + private ArrayDeque videoFrameQueue = new ArrayDeque<>(); + + /** + * Internal data structure used to hold decoded AudioFrames and the associated timestamp. + */ + private ArrayDeque audioFrameQueue = new ArrayDeque<>(); + + private AVFormatContext pFormatCtx; + + private int videoStream = -1; + private int audioStream = -1; + private avcodec.AVCodecContext pCodecCtxVideo = null; + private avcodec.AVCodecContext pCodecCtxAudio = null; + + /** + * Field for raw frame as returned by decoder (regardless of being audio or video). + */ + private avutil.AVFrame pFrame = null; + + /** + * Field for RGB frame (decoded video frame). + */ + private avutil.AVFrame pFrameRGB = null; + + /** + * Field for re-sampled audio-sample. + */ + private avutil.AVFrame resampledFrame = null; + + private avcodec.AVPacket packet; + private BytePointer buffer = null; + + private IntPointer out_linesize = new IntPointer(); + + private swscale.SwsContext sws_ctx = null; + private swresample.SwrContext swr_ctx = null; + + private VideoDescriptor videoDescriptor = null; + private AudioDescriptor audioDescriptor = null; + private SubTitleDecoder subtitles = null; + + /** + * Indicates that decoding of video-data is complete. + */ + private final AtomicBoolean videoComplete = new AtomicBoolean(false); + + /** + * Indicates that decoding of video-data is complete. + */ + private final AtomicBoolean audioComplete = new AtomicBoolean(false); + + /** + * Indicates the EOF has been reached during decoding. + */ + private final AtomicBoolean eof = new AtomicBoolean(false); + + /** + * The {@link CachedDataFactory} reference used to create {@link MultiImage} objects. + */ + private CachedDataFactory factory; + + private boolean readFrame(boolean queue) { + /* Tries to read a new packet from the stream. */ + int read_results = avformat.av_read_frame(this.pFormatCtx, this.packet); + if (read_results < 0 && !(read_results == avutil.AVERROR_EOF)) { + LOGGER.error("Error occurred while reading packet. FFMPEG av_read_frame() returned code {}.", read_results); + avcodec.av_packet_unref(this.packet); + return false; + } else if (read_results == avutil.AVERROR_EOF && this.eof.getAndSet(true)) { + return false; } - private byte[] bytes; - private int[] pixels; - - /** Internal data structure used to hold decoded VideoFrames and the associated timestamp. */ - private ArrayDeque videoFrameQueue = new ArrayDeque<>(); - - /** Internal data structure used to hold decoded AudioFrames and the associated timestamp. */ - private ArrayDeque audioFrameQueue = new ArrayDeque<>(); - - private AVFormatContext pFormatCtx; - - private int videoStream = -1; - private int audioStream = -1; - private avcodec.AVCodecContext pCodecCtxVideo = null; - private avcodec.AVCodecContext pCodecCtxAudio = null; - - /** Field for raw frame as returned by decoder (regardless of being audio or video). */ - private avutil.AVFrame pFrame = null; - - /** Field for RGB frame (decoded video frame). */ - private avutil.AVFrame pFrameRGB = null; - - /** Field for re-sampled audio-sample. */ - private avutil.AVFrame resampledFrame = null; - - private avcodec.AVPacket packet; - private BytePointer buffer = null; - - private IntPointer out_linesize = new IntPointer(); - - private swscale.SwsContext sws_ctx = null; - private swresample.SwrContext swr_ctx = null; - - private VideoDescriptor videoDescriptor = null; - private AudioDescriptor audioDescriptor = null; - private SubTitleDecoder subtitles = null; - - /** Indicates that decoding of video-data is complete. */ - private final AtomicBoolean videoComplete = new AtomicBoolean(false); - - /** Indicates that decoding of video-data is complete. */ - private final AtomicBoolean audioComplete = new AtomicBoolean(false); - - /** Indicates the EOF has been reached during decoding. */ - private final AtomicBoolean eof = new AtomicBoolean(false); - - /** The {@link CachedDataFactory} reference used to create {@link MultiImage} objects. */ - private CachedDataFactory factory; - - private boolean readFrame(boolean queue) { - /* Tries to read a new packet from the stream. */ - int read_results = avformat.av_read_frame(this.pFormatCtx, this.packet); - if (read_results < 0 && !(read_results == avutil.AVERROR_EOF)) { - LOGGER.error("Error occurred while reading packet. FFMPEG av_read_frame() returned code {}.", read_results); - avcodec.av_packet_unref(this.packet); - return false; - } else if (read_results == avutil.AVERROR_EOF && this.eof.getAndSet(true)) { - return false; - } - - /* Two cases: Audio or video stream!. */ - if (packet.stream_index() == this.videoStream) { - /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ - int decode_results = avcodec.avcodec_send_packet(this.pCodecCtxVideo, this.eof.get() ? null : this.packet); - if (decode_results < 0) { - LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); - avcodec.av_packet_unref(this.packet); - return true; - } - - /* - * Because a packet can theoretically contain more than one frame; repeat decoding until no samples are - * remaining in the packet. - */ - while (avcodec.avcodec_receive_frame(this.pCodecCtxVideo, this.pFrame) == 0) { - /* If queue is true; enqueue frame. */ - if (queue) { - readVideo(); - } - } - } else if (packet.stream_index() == this.audioStream) { - /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ - int decode_results = avcodec.avcodec_send_packet(this.pCodecCtxAudio, this.eof.get() ? null : this.packet); - if (decode_results < 0) { - LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); - avcodec.av_packet_unref(this.packet); - return true; - } - - /* - * Because a packet can theoretically contain more than one frame; repeat decoding until no samples are remaining in the packet. - */ - while (avcodec.avcodec_receive_frame(this.pCodecCtxAudio, this.pFrame) == 0) { - /* If queue is true; enqueue frame. */ - if (queue) { - if (this.swr_ctx != null) { - this.enqueueResampledAudio(); - } else { - this.enqueueOriginalAudio(); - } - } - } - } - - /* Free the packet that was allocated by av_read_frame. */ + /* Two cases: Audio or video stream!. */ + if (packet.stream_index() == this.videoStream) { + /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ + int decode_results = avcodec.avcodec_send_packet(this.pCodecCtxVideo, this.eof.get() ? null : this.packet); + if (decode_results < 0) { + LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); avcodec.av_packet_unref(this.packet); - return true; + } + + /* + * Because a packet can theoretically contain more than one frame; repeat decoding until no samples are + * remaining in the packet. + */ + while (avcodec.avcodec_receive_frame(this.pCodecCtxVideo, this.pFrame) == 0) { + /* If queue is true; enqueue frame. */ + if (queue) { + readVideo(); + } + } + } else if (packet.stream_index() == this.audioStream) { + /* Send packet to decoder. If no packet was read, send null to flush the buffer. */ + int decode_results = avcodec.avcodec_send_packet(this.pCodecCtxAudio, this.eof.get() ? null : this.packet); + if (decode_results < 0) { + LOGGER.error("Error occurred while decoding frames from packet. FFMPEG avcodec_send_packet() returned code {}.", decode_results); + avcodec.av_packet_unref(this.packet); + return true; + } + + /* + * Because a packet can theoretically contain more than one frame; repeat decoding until no samples are remaining in the packet. + */ + while (avcodec.avcodec_receive_frame(this.pCodecCtxAudio, this.pFrame) == 0) { + /* If queue is true; enqueue frame. */ + if (queue) { + if (this.swr_ctx != null) { + this.enqueueResampledAudio(); + } else { + this.enqueueOriginalAudio(); + } + } + } } - /** - * Returns the timestamp in milliseconds of the currently active frame. That timestamp is based on - * a best-effort calculation by the FFMPEG decoder. - * - * @param stream Number of the stream. Determines the time base used to calculate the timestamp; - * @return Timestamp of the current frame. - */ - private Long getFrameTimestamp(int stream) { - AVRational timebase = this.pFormatCtx.streams(stream).time_base(); - return Math.floorDiv((this.pFrame.best_effort_timestamp() * timebase.num() * 1000), (long)timebase.den()); - } - - /** - * Reads the decoded audio frames copies them directly into the AudioFrame data-structure which is subsequently - * enqueued. - */ - private void enqueueOriginalAudio() { - /* Allocate output buffer... */ - int buffersize = this.pFrame.nb_samples() * avutil.av_get_bytes_per_sample(this.pFrame.format()) * this.pFrame.channels(); - byte[] buffer = new byte[buffersize]; - this.pFrame.data(0).position(0).get(buffer); - - /* Prepare frame and associated timestamp and add it to output queue. */ - AudioFrame frame = new AudioFrame(this.pCodecCtxAudio.frame_number(), this.getFrameTimestamp(this.audioStream), buffer, this.audioDescriptor); - this.audioFrameQueue.add(frame); + /* Free the packet that was allocated by av_read_frame. */ + avcodec.av_packet_unref(this.packet); + + return true; + } + + /** + * Returns the timestamp in milliseconds of the currently active frame. That timestamp is based on a best-effort calculation by the FFMPEG decoder. + * + * @param stream Number of the stream. Determines the time base used to calculate the timestamp; + * @return Timestamp of the current frame. + */ + private Long getFrameTimestamp(int stream) { + AVRational timebase = this.pFormatCtx.streams(stream).time_base(); + return Math.floorDiv((this.pFrame.best_effort_timestamp() * timebase.num() * 1000), (long) timebase.den()); + } + + /** + * Reads the decoded audio frames copies them directly into the AudioFrame data-structure which is subsequently enqueued. + */ + private void enqueueOriginalAudio() { + /* Allocate output buffer... */ + int buffersize = this.pFrame.nb_samples() * avutil.av_get_bytes_per_sample(this.pFrame.format()) * this.pFrame.channels(); + byte[] buffer = new byte[buffersize]; + this.pFrame.data(0).position(0).get(buffer); + + /* Prepare frame and associated timestamp and add it to output queue. */ + AudioFrame frame = new AudioFrame(this.pCodecCtxAudio.frame_number(), this.getFrameTimestamp(this.audioStream), buffer, this.audioDescriptor); + this.audioFrameQueue.add(frame); + } + + + /** + * Reads the decoded audio frames and re-samples them using the SWR-CTX. The re-sampled frames are then copied into the AudioFrame data-structure, which is subsequently enqueued. + */ + private void enqueueResampledAudio() { + /* Convert decoded frame. Break if resampling fails.*/ + if (swresample.swr_convert(this.swr_ctx, null, 0, this.pFrame.data(), this.pFrame.nb_samples()) < 0) { + LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName()); + return; } - - /** - * Reads the decoded audio frames and re-samples them using the SWR-CTX. The re-sampled frames are then - * copied into the AudioFrame data-structure, which is subsequently enqueued. - */ - private void enqueueResampledAudio() { - /* Convert decoded frame. Break if resampling fails.*/ - if (swresample.swr_convert(this.swr_ctx, null, 0, this.pFrame.data(), this.pFrame.nb_samples()) < 0) { - LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName()); - return; - } - - /* Prepare ByteOutputStream to write resampled data to. */ - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - - /* Now drain buffer and write samples to queue. */ - while(true) { - /* Estimate number of samples that were converted. */ - int out_samples = swresample.swr_get_out_samples(this.swr_ctx, 0); - if (out_samples < BYTES_PER_SAMPLE * this.resampledFrame.channels()) { - break; - } - - /* Allocate output frame and read converted samples. If no sample was read -> break (draining completed). */ - avutil.av_samples_alloc(this.resampledFrame.data(), out_linesize, this.resampledFrame.channels(), out_samples, TARGET_FORMAT, 0); - out_samples = swresample.swr_convert(this.swr_ctx, this.resampledFrame.data(), out_samples, null, 0); - if (out_samples == 0) { - break; - } - - /* Allocate output buffer... */ - int buffersize = out_samples * BYTES_PER_SAMPLE * this.resampledFrame.channels(); - byte[] buffer = new byte[buffersize]; - this.resampledFrame.data(0).position(0).get(buffer); - try { - stream.write(buffer); - } catch (IOException e) { - LOGGER.error("Could not write re-sampled frame to ByteArrayOutputStream due to an exception ({}).", LogHelper.getStackTrace(e)); - break; - } - } - - /* Prepare frame and associated timestamp and add it to output queue. */ - AudioFrame frame = new AudioFrame(this.pCodecCtxAudio.frame_number(), this.getFrameTimestamp(this.audioStream), stream.toByteArray(), this.audioDescriptor); - this.audioFrameQueue.add(frame); + /* Prepare ByteOutputStream to write resampled data to. */ + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + /* Now drain buffer and write samples to queue. */ + while (true) { + /* Estimate number of samples that were converted. */ + int out_samples = swresample.swr_get_out_samples(this.swr_ctx, 0); + if (out_samples < BYTES_PER_SAMPLE * this.resampledFrame.channels()) { + break; + } + + /* Allocate output frame and read converted samples. If no sample was read -> break (draining completed). */ + avutil.av_samples_alloc(this.resampledFrame.data(), out_linesize, this.resampledFrame.channels(), out_samples, TARGET_FORMAT, 0); + out_samples = swresample.swr_convert(this.swr_ctx, this.resampledFrame.data(), out_samples, null, 0); + if (out_samples == 0) { + break; + } + + /* Allocate output buffer... */ + int buffersize = out_samples * BYTES_PER_SAMPLE * this.resampledFrame.channels(); + byte[] buffer = new byte[buffersize]; + this.resampledFrame.data(0).position(0).get(buffer); + try { + stream.write(buffer); + } catch (IOException e) { + LOGGER.error("Could not write re-sampled frame to ByteArrayOutputStream due to an exception ({}).", LogHelper.getStackTrace(e)); + break; + } } - /** - * Reads the decoded video frames and re-sizes them. The re-sized video frame is then copied into a VideoFrame data - * structure, which is subsequently enqueued. - */ - private void readVideo() { - // Convert the image from its native format to RGB - swscale.sws_scale(this.sws_ctx, this.pFrame.data(), this.pFrame.linesize(), 0, this.pCodecCtxVideo.height(), this.pFrameRGB.data(), this.pFrameRGB.linesize()); + /* Prepare frame and associated timestamp and add it to output queue. */ + AudioFrame frame = new AudioFrame(this.pCodecCtxAudio.frame_number(), this.getFrameTimestamp(this.audioStream), stream.toByteArray(), this.audioDescriptor); + this.audioFrameQueue.add(frame); + } - // Write pixel data - BytePointer data = this.pFrameRGB.data(0); + /** + * Reads the decoded video frames and re-sizes them. The re-sized video frame is then copied into a VideoFrame data structure, which is subsequently enqueued. + */ + private void readVideo() { + // Convert the image from its native format to RGB + swscale.sws_scale(this.sws_ctx, this.pFrame.data(), this.pFrame.linesize(), 0, this.pCodecCtxVideo.height(), this.pFrameRGB.data(), this.pFrameRGB.linesize()); - data.position(0).get(bytes); + // Write pixel data + BytePointer data = this.pFrameRGB.data(0); - for (int i = 0; i < pixels.length; ++i) { - int pos = 3 * i; - int r = bytes[pos] & 0xff; - int g = bytes[pos + 1] & 0xff; - int b = bytes[pos + 2] & 0xff; + data.position(0).get(bytes); - pixels[i] = ((0xFF) << 24) | ((r & 0xFF) << 16) | ((g & 0xFF) << 8) | ((b & 0xFF) << 0); - } + for (int i = 0; i < pixels.length; ++i) { + int pos = 3 * i; + int r = bytes[pos] & 0xff; + int g = bytes[pos + 1] & 0xff; + int b = bytes[pos + 2] & 0xff; - /* Prepare frame and associated timestamp and add it to output queue. */ - VideoFrame videoFrame = new VideoFrame(this.pCodecCtxVideo.frame_number(), this.getFrameTimestamp(this.videoStream), this.factory.newMultiImage(this.videoDescriptor.getWidth(), this.videoDescriptor.getHeight(), pixels), this.videoDescriptor); - this.videoFrameQueue.add(videoFrame); + pixels[i] = ((0xFF) << 24) | ((r & 0xFF) << 16) | ((g & 0xFF) << 8) | ((b & 0xFF) << 0); } - /** - * Closes the FFMpegVideoDecoder and frees all (non-native) resources associated with it. - */ - @Override - public void close() { - if (this.pFormatCtx == null) { - return; - } - - /* Free the raw frame. */ - if (this.pFrame != null) { - avutil.av_frame_free(this.pFrame); - this.pFrame = null; - } - - /* Free the frame holding re-sampled audio. */ - if (this.resampledFrame != null) { - avutil.av_frame_free(this.resampledFrame); - this.resampledFrame = null; - } - - /* Free the frame holding re-sized video. */ - if (this.pFrameRGB != null) { - avutil.av_frame_free(this.pFrameRGB); - this.pFrameRGB = null; - } - - /* Free the packet. */ - if (this.packet != null) { - avcodec.av_packet_free(this.packet); - this.packet.close(); - this.packet = null; - } - - /* Frees the SWR context. */ - if (this.swr_ctx != null) { - swresample.swr_free(this.swr_ctx); - this.swr_ctx = null; - } - - /* Frees the SWS context. */ - if (this.sws_ctx != null) { - swscale.sws_freeContext(this.sws_ctx); - this.sws_ctx = null; - } + /* Prepare frame and associated timestamp and add it to output queue. */ + VideoFrame videoFrame = new VideoFrame(this.pCodecCtxVideo.frame_number(), this.getFrameTimestamp(this.videoStream), this.factory.newMultiImage(this.videoDescriptor.getWidth(), this.videoDescriptor.getHeight(), pixels), this.videoDescriptor); + this.videoFrameQueue.add(videoFrame); + } + + /** + * Closes the FFMpegVideoDecoder and frees all (non-native) resources associated with it. + */ + @Override + public void close() { + if (this.pFormatCtx == null) { + return; + } - /* Closes the audio codec context. */ - if (this.pCodecCtxAudio != null) { - avcodec.avcodec_free_context(this.pCodecCtxAudio); - this.pCodecCtxAudio = null; - } + /* Free the raw frame. */ + if (this.pFrame != null) { + avutil.av_frame_free(this.pFrame); + this.pFrame = null; + } - /* Closes the audio codec context. */ - if (this.pCodecCtxVideo != null) { - avcodec.avcodec_free_context(this.pCodecCtxVideo); - this.pCodecCtxVideo = null; - } + /* Free the frame holding re-sampled audio. */ + if (this.resampledFrame != null) { + avutil.av_frame_free(this.resampledFrame); + this.resampledFrame = null; + } - /* Frees the ByteBuffer used to resize images. */ - if (this.buffer != null) { - avutil.av_free(this.buffer); - this.buffer = null; - } + /* Free the frame holding re-sized video. */ + if (this.pFrameRGB != null) { + avutil.av_frame_free(this.pFrameRGB); + this.pFrameRGB = null; + } - /* Close the audio file context. */ - avformat.avformat_close_input(this.pFormatCtx); - this.pFormatCtx = null; + /* Free the packet. */ + if (this.packet != null) { + avcodec.av_packet_free(this.packet); + this.packet.close(); + this.packet = null; } - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from - * the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - if(!Files.exists(path)){ - LOGGER.error("File does not exist {}", path.toString()); - return false; - } + /* Frees the SWR context. */ + if (this.swr_ctx != null) { + swresample.swr_free(this.swr_ctx); + this.swr_ctx = null; + } - /* Initialize MultiImageFactory using the ImageCacheConfig. */ - if (cacheConfig == null) { - LOGGER.error("You must provide a valid ImageCacheConfig when initializing the FFMpegVideoDecoder."); - return false; - } - this.factory = cacheConfig.sharedCachedDataFactory(); + /* Frees the SWS context. */ + if (this.sws_ctx != null) { + swscale.sws_freeContext(this.sws_ctx); + this.sws_ctx = null; + } - /* Initialize the AVFormatContext. */ - this.pFormatCtx = avformat.avformat_alloc_context(); + /* Closes the audio codec context. */ + if (this.pCodecCtxAudio != null) { + avcodec.avcodec_free_context(this.pCodecCtxAudio); + this.pCodecCtxAudio = null; + } - /* Open video file. */ - if (avformat.avformat_open_input(this.pFormatCtx, path.toString(), null, null) != 0) { - LOGGER.error("Error while accessing file {}", path.toString()); - return false; - } + /* Closes the audio codec context. */ + if (this.pCodecCtxVideo != null) { + avcodec.avcodec_free_context(this.pCodecCtxVideo); + this.pCodecCtxVideo = null; + } - /* Retrieve stream information. */ - if (avformat.avformat_find_stream_info(pFormatCtx, (PointerPointer)null) < 0) { - LOGGER.error("Error, Couldn't find stream information"); - return false; - } + /* Frees the ByteBuffer used to resize images. */ + if (this.buffer != null) { + avutil.av_free(this.buffer); + this.buffer = null; + } - /* Initialize the packet. */ - this.packet = avcodec.av_packet_alloc(); - if (this.packet == null) { - LOGGER.error("Could not allocate packet data structure for decoded data."); - return false; - } + /* Close the audio file context. */ + avformat.avformat_close_input(this.pFormatCtx); + this.pFormatCtx = null; + } + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + if (!Files.exists(path)) { + LOGGER.error("File does not exist {}", path.toString()); + return false; + } - /* Allocate frame that holds decoded frame information. */ - this.pFrame = avutil.av_frame_alloc(); - if (this.pFrame == null) { - LOGGER.error("Could not allocate frame data structure for decoded data."); - return false; - } + /* Initialize MultiImageFactory using the ImageCacheConfig. */ + if (cacheConfig == null) { + LOGGER.error("You must provide a valid ImageCacheConfig when initializing the FFMpegVideoDecoder."); + return false; + } + this.factory = cacheConfig.sharedCachedDataFactory(); - /* Open subtitles file (if configured). */ - if (decoderConfig.namedAsBoolean(CONFIG_SUBTITLE_PROPERTY, false)) { - final Optional subtitles = SubtitleDecoderFactory.subtitleForVideo(path); - subtitles.ifPresent(subTitleDecoder -> this.subtitles = subTitleDecoder); - } + /* Initialize the AVFormatContext. */ + this.pFormatCtx = avformat.avformat_alloc_context(); - if (this.initVideo(decoderConfig) && this.initAudio(decoderConfig)) { - LOGGER.debug("{} was initialized successfully.", this.getClass().getName()); - return true; - } else { - return false; - } + /* Open video file. */ + if (avformat.avformat_open_input(this.pFormatCtx, path.toString(), null, null) != 0) { + LOGGER.error("Error while accessing file {}", path.toString()); + return false; } - /** - * Initializes the video decoding part of FFMPEG. - * - * @param config The {@link DecoderConfig} used for configuring the {@link FFMpegVideoDecoder}. - * @return True if vide decoder was initialized, false otherwise. - */ - private boolean initVideo(DecoderConfig config) { - /* Read decoder config (VIDEO). */ - int maxWidth = config.namedAsInt(CONFIG_MAXWIDTH_PROPERTY, CONFIG_MAXWIDTH_DEFAULT); - int maxHeight = config.namedAsInt(CONFIG_HEIGHT_PROPERTY, CONFIG_MAXHEIGHT_DEFAULT); - - /* Find the best video stream. */ - final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); - this.videoStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_VIDEO,-1, -1, codec, 0); - if (this.videoStream == -1) { - LOGGER.error("Couldn't find a video stream."); - return false; - } + /* Retrieve stream information. */ + if (avformat.avformat_find_stream_info(pFormatCtx, (PointerPointer) null) < 0) { + LOGGER.error("Error, Couldn't find stream information"); + return false; + } - /* Allocate new codec-context for codec returned by av_find_best_stream(). */ - this.pCodecCtxVideo = avcodec.avcodec_alloc_context3(codec); - avcodec.avcodec_parameters_to_context(this.pCodecCtxVideo, this.pFormatCtx.streams(this.videoStream).codecpar()); - /* Open the code context. */ - if (avcodec.avcodec_open2(this.pCodecCtxVideo, codec, (AVDictionary)null) < 0) { - LOGGER.error("Error, Could not open video codec."); - return false; - } + /* Initialize the packet. */ + this.packet = avcodec.av_packet_alloc(); + if (this.packet == null) { + LOGGER.error("Could not allocate packet data structure for decoded data."); + return false; + } - /* Allocate an AVFrame structure that will hold the resized video. */ - this.pFrameRGB = avutil.av_frame_alloc(); - if(pFrameRGB == null) { - LOGGER.error("Error. Could not allocate frame for resized video."); - return false; - } + /* Allocate frame that holds decoded frame information. */ + this.pFrame = avutil.av_frame_alloc(); + if (this.pFrame == null) { + LOGGER.error("Could not allocate frame data structure for decoded data."); + return false; + } - int originalWidth = pCodecCtxVideo.width(); - int originalHeight = pCodecCtxVideo.height(); - int width = originalWidth; - int height = originalHeight; + /* Open subtitles file (if configured). */ + if (decoderConfig.namedAsBoolean(CONFIG_SUBTITLE_PROPERTY, false)) { + final Optional subtitles = SubtitleDecoderFactory.subtitleForVideo(path); + subtitles.ifPresent(subTitleDecoder -> this.subtitles = subTitleDecoder); + } - if (originalWidth > maxWidth || originalHeight > maxHeight) { - float scaleDown = Math.min((float) maxWidth / (float) originalWidth, (float) maxHeight / (float) originalHeight); - width = Math.round(originalWidth * scaleDown); - height = Math.round(originalHeight * scaleDown); - LOGGER.debug("scaling input video down by a factor of {} from {}x{} to {}x{}", scaleDown, originalWidth, originalHeight, width, height); - } + if (this.initVideo(decoderConfig) && this.initAudio(decoderConfig)) { + LOGGER.debug("{} was initialized successfully.", this.getClass().getName()); + return true; + } else { + return false; + } + } + + /** + * Initializes the video decoding part of FFMPEG. + * + * @param config The {@link DecoderConfig} used for configuring the {@link FFMpegVideoDecoder}. + * @return True if vide decoder was initialized, false otherwise. + */ + private boolean initVideo(DecoderConfig config) { + /* Read decoder config (VIDEO). */ + int maxWidth = config.namedAsInt(CONFIG_MAXWIDTH_PROPERTY, CONFIG_MAXWIDTH_DEFAULT); + int maxHeight = config.namedAsInt(CONFIG_HEIGHT_PROPERTY, CONFIG_MAXHEIGHT_DEFAULT); + + /* Find the best video stream. */ + final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); + this.videoStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_VIDEO, -1, -1, codec, 0); + if (this.videoStream == -1) { + LOGGER.error("Couldn't find a video stream."); + return false; + } - bytes = new byte[width * height * 3]; - pixels = new int[width * height]; + /* Allocate new codec-context for codec returned by av_find_best_stream(). */ + this.pCodecCtxVideo = avcodec.avcodec_alloc_context3(codec); + avcodec.avcodec_parameters_to_context(this.pCodecCtxVideo, this.pFormatCtx.streams(this.videoStream).codecpar()); + /* Open the code context. */ + if (avcodec.avcodec_open2(this.pCodecCtxVideo, codec, (AVDictionary) null) < 0) { + LOGGER.error("Error, Could not open video codec."); + return false; + } - /* Initialize data-structures used for resized image. */ - int numBytes = avutil.av_image_get_buffer_size(avutil.AV_PIX_FMT_RGB24, pCodecCtxVideo.width(), pCodecCtxVideo.height(), 1); - this.buffer = new BytePointer(avutil.av_malloc(numBytes)); - avutil.av_image_fill_arrays(this.pFrameRGB.data(), this.pFrameRGB.linesize(), this.buffer, avutil.AV_PIX_FMT_RGB24, width, height, 1); + /* Allocate an AVFrame structure that will hold the resized video. */ + this.pFrameRGB = avutil.av_frame_alloc(); + if (pFrameRGB == null) { + LOGGER.error("Error. Could not allocate frame for resized video."); + return false; + } - /* Initialize SWS Context. */ - this.sws_ctx = swscale.sws_getContext(this.pCodecCtxVideo.width(), this.pCodecCtxVideo.height(), this.pCodecCtxVideo.pix_fmt(), width, height, avutil.AV_PIX_FMT_RGB24, swscale.SWS_BILINEAR, null, null, (DoublePointer)null); + int originalWidth = pCodecCtxVideo.width(); + int originalHeight = pCodecCtxVideo.height(); + int width = originalWidth; + int height = originalHeight; - /* Initialize VideoDescriptor. */ - AVRational timebase = this.pFormatCtx.streams(this.videoStream).time_base(); - long duration = (1000L * timebase.num() * this.pFormatCtx.streams(this.videoStream).duration()/timebase.den()); - AVRational framerate = this.pFormatCtx.streams(this.videoStream).avg_frame_rate(); - float fps = ((float) framerate.num()) / ((float)framerate.den()); - this.videoDescriptor = new VideoDescriptor(fps, duration, width, height); + if (originalWidth > maxWidth || originalHeight > maxHeight) { + float scaleDown = Math.min((float) maxWidth / (float) originalWidth, (float) maxHeight / (float) originalHeight); + width = Math.round(originalWidth * scaleDown); + height = Math.round(originalHeight * scaleDown); + LOGGER.debug("scaling input video down by a factor of {} from {}x{} to {}x{}", scaleDown, originalWidth, originalHeight, width, height); + } - /* Return true (success). */ - return true; + bytes = new byte[width * height * 3]; + pixels = new int[width * height]; + + /* Initialize data-structures used for resized image. */ + int numBytes = avutil.av_image_get_buffer_size(avutil.AV_PIX_FMT_RGB24, pCodecCtxVideo.width(), pCodecCtxVideo.height(), 1); + this.buffer = new BytePointer(avutil.av_malloc(numBytes)); + avutil.av_image_fill_arrays(this.pFrameRGB.data(), this.pFrameRGB.linesize(), this.buffer, avutil.AV_PIX_FMT_RGB24, width, height, 1); + + /* Initialize SWS Context. */ + this.sws_ctx = swscale.sws_getContext(this.pCodecCtxVideo.width(), this.pCodecCtxVideo.height(), this.pCodecCtxVideo.pix_fmt(), width, height, avutil.AV_PIX_FMT_RGB24, swscale.SWS_BILINEAR, null, null, (DoublePointer) null); + + /* Initialize VideoDescriptor. */ + AVRational timebase = this.pFormatCtx.streams(this.videoStream).time_base(); + long duration = (1000L * timebase.num() * this.pFormatCtx.streams(this.videoStream).duration() / timebase.den()); + AVRational framerate = this.pFormatCtx.streams(this.videoStream).avg_frame_rate(); + float fps = ((float) framerate.num()) / ((float) framerate.den()); + this.videoDescriptor = new VideoDescriptor(fps, duration, width, height); + + /* Return true (success). */ + return true; + } + + /** + * Initializes the audio decoding part of FFMPEG. + * + * @param config The {@link DecoderConfig} used for configuring the {@link FFMpegVideoDecoder}. + * @return True if a) audio decoder was initialized, b) number of channels is smaller than zero (no audio) or c) audio is unavailable or unsupported, false if initialization failed due to technical reasons. + */ + private boolean initAudio(DecoderConfig config) { + /* Read decoder configuration. */ + int samplerate = config.namedAsInt(CONFIG_SAMPLERATE_PROPERTY, CONFIG_SAMPLERATE_DEFAULT); + int channels = config.namedAsInt(CONFIG_CHANNELS_PROPERTY, CONFIG_CHANNELS_DEFAULT); + long channellayout = avutil.av_get_default_channel_layout(channels); + + /* If number of channels is smaller or equal than zero; return true (no audio decoded). */ + if (channels <= 0) { + LOGGER.info("Channel setting is smaller than zero. Continuing without audio!"); + this.audioComplete.set(true); + return true; } - /** - * Initializes the audio decoding part of FFMPEG. - * - * @param config The {@link DecoderConfig} used for configuring the {@link FFMpegVideoDecoder}. - * @return True if a) audio decoder was initialized, b) number of channels is smaller than zero (no audio) or c) audio is unavailable or unsupported, false if initialization failed due to technical reasons. - */ - private boolean initAudio(DecoderConfig config) { - /* Read decoder configuration. */ - int samplerate = config.namedAsInt(CONFIG_SAMPLERATE_PROPERTY, CONFIG_SAMPLERATE_DEFAULT); - int channels = config.namedAsInt(CONFIG_CHANNELS_PROPERTY, CONFIG_CHANNELS_DEFAULT); - long channellayout = avutil.av_get_default_channel_layout(channels); - - /* If number of channels is smaller or equal than zero; return true (no audio decoded). */ - if (channels <= 0) { - LOGGER.info("Channel setting is smaller than zero. Continuing without audio!"); - this.audioComplete.set(true); - return true; - } + /* Find the best frames stream. */ + final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); + this.audioStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_AUDIO, -1, -1, codec, 0); + if (this.audioStream < 0) { + LOGGER.warn("Couldn't find a supported audio stream. Continuing without audio!"); + this.audioComplete.set(true); + return true; + } - /* Find the best frames stream. */ - final AVCodec codec = avcodec.av_codec_iterate(new Pointer()); - this.audioStream = avformat.av_find_best_stream(this.pFormatCtx, avutil.AVMEDIA_TYPE_AUDIO,-1, -1, codec, 0); - if (this.audioStream < 0) { - LOGGER.warn("Couldn't find a supported audio stream. Continuing without audio!"); - this.audioComplete.set(true); - return true; - } + /* Allocate new codec-context. */ + this.pCodecCtxAudio = avcodec.avcodec_alloc_context3(codec); + avcodec.avcodec_parameters_to_context(this.pCodecCtxAudio, this.pFormatCtx.streams(this.audioStream).codecpar()); - /* Allocate new codec-context. */ - this.pCodecCtxAudio = avcodec.avcodec_alloc_context3(codec); - avcodec.avcodec_parameters_to_context(this.pCodecCtxAudio, this.pFormatCtx.streams(this.audioStream).codecpar()); + /* Open the code context. */ + if (avcodec.avcodec_open2(this.pCodecCtxAudio, codec, (AVDictionary) null) < 0) { + LOGGER.error("Could not open audio codec. Continuing without audio!"); + this.audioComplete.set(true); + return true; + } - /* Open the code context. */ - if (avcodec.avcodec_open2(this.pCodecCtxAudio, codec, (AVDictionary)null) < 0) { - LOGGER.error("Could not open audio codec. Continuing without audio!"); - this.audioComplete.set(true); - return true; - } + /* Allocate the re-sample context. */ + this.swr_ctx = swresample.swr_alloc_set_opts(null, channellayout, TARGET_FORMAT, samplerate, this.pCodecCtxAudio.channel_layout(), this.pCodecCtxAudio.sample_fmt(), this.pCodecCtxAudio.sample_rate(), 0, null); + if (swresample.swr_init(this.swr_ctx) < 0) { + this.swr_ctx = null; + LOGGER.warn("Could not open re-sample context - original format will be kept!"); + } - /* Allocate the re-sample context. */ - this.swr_ctx = swresample.swr_alloc_set_opts(null, channellayout, TARGET_FORMAT, samplerate, this.pCodecCtxAudio.channel_layout(), this.pCodecCtxAudio.sample_fmt(), this.pCodecCtxAudio.sample_rate(), 0, null); - if(swresample.swr_init(this.swr_ctx) < 0) { - this.swr_ctx = null; - LOGGER.warn("Could not open re-sample context - original format will be kept!"); - } + /* Initialize decoded and resampled frame. */ + this.resampledFrame = avutil.av_frame_alloc(); + if (this.resampledFrame == null) { + LOGGER.error("Could not allocate frame data structure for re-sampled data."); + return false; + } - /* Initialize decoded and resampled frame. */ - this.resampledFrame = avutil.av_frame_alloc(); - if (this.resampledFrame == null) { - LOGGER.error("Could not allocate frame data structure for re-sampled data."); - return false; - } + /* Initialize out-frame. */ + this.resampledFrame = avutil.av_frame_alloc(); + this.resampledFrame.channel_layout(channellayout); + this.resampledFrame.sample_rate(samplerate); + this.resampledFrame.channels(channels); + this.resampledFrame.format(TARGET_FORMAT); + + /* Initialize the AudioDescriptor. */ + final AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); + final long duration = (1000L * timebase.num() * this.pFormatCtx.streams(this.audioStream).duration() / timebase.den()); + if (this.swr_ctx == null) { + this.audioDescriptor = new AudioDescriptor(this.pCodecCtxAudio.sample_rate(), this.pCodecCtxAudio.channels(), duration); + } else { + this.audioDescriptor = new AudioDescriptor(this.resampledFrame.sample_rate(), this.resampledFrame.channels(), duration); + } - /* Initialize out-frame. */ - this.resampledFrame = avutil.av_frame_alloc(); - this.resampledFrame.channel_layout(channellayout); - this.resampledFrame.sample_rate(samplerate); - this.resampledFrame.channels(channels); - this.resampledFrame.format(TARGET_FORMAT); - - /* Initialize the AudioDescriptor. */ - final AVRational timebase = this.pFormatCtx.streams(this.audioStream).time_base(); - final long duration = (1000L * timebase.num() * this.pFormatCtx.streams(this.audioStream).duration()/timebase.den()); - if (this.swr_ctx == null) { - this.audioDescriptor = new AudioDescriptor(this.pCodecCtxAudio.sample_rate(), this.pCodecCtxAudio.channels(), duration); - } else { - this.audioDescriptor = new AudioDescriptor(this.resampledFrame.sample_rate(), this.resampledFrame.channels(), duration); - } + /* Completed initialization. */ + return true; + } + + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public VideoFrame getNext() { + /* Read frames until a video-frame becomes available. */ + while (this.videoFrameQueue.isEmpty() && !this.eof.get()) { + this.readFrame(true); + } - /* Completed initialization. */ - return true; + /* If frame-queue is empty and EOF has been reached, then video decoding was completed. */ + if (this.videoFrameQueue.isEmpty()) { + this.videoComplete.set(true); + return null; } + /* Fetch that video frame. */ + final VideoFrame videoFrame = this.videoFrameQueue.poll(); - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until - * complete() returns false. From which on this method will return null. - * - * @return Content of type T. + /* + * Now if the audio stream is set, read AudioFrames until the timestamp of the next AudioFrame in the + * queue becomes greater than the timestamp of the VideoFrame. All these AudioFrames go to the VideoFrame. */ - @Override - public VideoFrame getNext() { - /* Read frames until a video-frame becomes available. */ - while (this.videoFrameQueue.isEmpty() && !this.eof.get()) { - this.readFrame(true); - } - - /* If frame-queue is empty and EOF has been reached, then video decoding was completed. */ - if (this.videoFrameQueue.isEmpty()) { - this.videoComplete.set(true); - return null; - } - - /* Fetch that video frame. */ - final VideoFrame videoFrame = this.videoFrameQueue.poll(); - - /* - * Now if the audio stream is set, read AudioFrames until the timestamp of the next AudioFrame in the - * queue becomes greater than the timestamp of the VideoFrame. All these AudioFrames go to the VideoFrame. - */ - while (!this.audioComplete.get()) { - while (this.audioFrameQueue.isEmpty() && !this.eof.get()) { - this.readFrame(true); - } - - if (this.audioFrameQueue.isEmpty()) { - this.audioComplete.set(true); - break; - } - - AudioFrame audioFrame = this.audioFrameQueue.peek(); - if (audioFrame.getTimestamp() <= videoFrame.getTimestamp()) { - videoFrame.addAudioFrame(this.audioFrameQueue.poll()); - } else { - break; - } - } - - /* - * Now, if there is a subtitle stream, try to append the corresponding subtitle. - */ - if (this.subtitles != null) { - while (true) { - final SubtitleItem item = this.subtitles.getLast(); - if (videoFrame.getTimestamp() >= item.getStartTimestamp() && videoFrame.getTimestamp() <= item.getEndTimestamp()) { - videoFrame.addSubtitleItem(item); - break; - } else if (videoFrame.getTimestamp() > item.getEndTimestamp()) { - if (!this.subtitles.increment()) break; - } else { - break; - } - } - } - - - /* Return VideoFrame. */ - return videoFrame; + while (!this.audioComplete.get()) { + while (this.audioFrameQueue.isEmpty() && !this.eof.get()) { + this.readFrame(true); + } + + if (this.audioFrameQueue.isEmpty()) { + this.audioComplete.set(true); + break; + } + + AudioFrame audioFrame = this.audioFrameQueue.peek(); + if (audioFrame.getTimestamp() <= videoFrame.getTimestamp()) { + videoFrame.addAudioFrame(this.audioFrameQueue.poll()); + } else { + break; + } } - /** - * Returns the total number of content pieces T this decoder can return for a given file. + /* + * Now, if there is a subtitle stream, try to append the corresponding subtitle. */ - @Override - public int count() { - return (int) this.pFormatCtx.streams(this.videoStream).nb_frames(); + if (this.subtitles != null) { + while (true) { + final SubtitleItem item = this.subtitles.getLast(); + if (videoFrame.getTimestamp() >= item.getStartTimestamp() && videoFrame.getTimestamp() <= item.getEndTimestamp()) { + videoFrame.addSubtitleItem(item); + break; + } else if (videoFrame.getTimestamp() > item.getEndTimestamp()) { + if (!this.subtitles.increment()) { + break; + } + } else { + break; + } + } } - /** - * Indicates whether or not the current decoder has more content to return or not. - * - * @return True if more content can be fetched, false otherwise. - */ - @Override - public boolean complete() { - return this.videoComplete.get(); - } - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; - } + /* Return VideoFrame. */ + return videoFrame; + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return (int) this.pFormatCtx.streams(this.videoStream).nb_frames(); + } + + /** + * Indicates whether or not the current decoder has more content to return or not. + * + * @return True if more content can be fetched, false otherwise. + */ + @Override + public boolean complete() { + return this.videoComplete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoEncoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoEncoder.java index 058bf7656..7ba78f90c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoEncoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoEncoder.java @@ -1,5 +1,21 @@ package org.vitrivr.cineast.core.extraction.decode.video; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_NONE; +import static org.bytedeco.javacpp.avformat.AVFMT_NOFILE; +import static org.bytedeco.javacpp.avformat.AVIOContext; +import static org.bytedeco.javacpp.avformat.AVIO_FLAG_WRITE; +import static org.bytedeco.javacpp.avformat.av_dump_format; +import static org.bytedeco.javacpp.avformat.av_write_trailer; +import static org.bytedeco.javacpp.avformat.avformat_alloc_output_context2; +import static org.bytedeco.javacpp.avformat.avformat_free_context; +import static org.bytedeco.javacpp.avformat.avformat_write_header; +import static org.bytedeco.javacpp.avformat.avio_closep; +import static org.bytedeco.javacpp.avformat.avio_open; +import static org.bytedeco.javacpp.avutil.AVDictionary; +import static org.bytedeco.javacpp.avutil.av_compare_ts; + +import java.util.LinkedList; +import java.util.Queue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bytedeco.javacpp.avformat; @@ -7,160 +23,150 @@ import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import java.util.LinkedList; -import java.util.Queue; - -import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_NONE; -import static org.bytedeco.javacpp.avformat.*; -import static org.bytedeco.javacpp.avutil.AVDictionary; -import static org.bytedeco.javacpp.avutil.av_compare_ts; - /** - * - * based on - * https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/muxing.c - * - * */ + * based on https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/muxing.c + */ public class FFMpegVideoEncoder { - private static final Logger LOGGER = LogManager.getLogger(); - - private AudioFrame tmpFrame = null; - - private VideoOutputStreamContainer video_st = null; - private AudioOutputStreamContainer audio_st = null; - private avformat.AVOutputFormat fmt; - private avformat.AVFormatContext oc = new avformat.AVFormatContext(); + private static final Logger LOGGER = LogManager.getLogger(); - private Queue imageQueue = new LinkedList<>(); - private Queue audioQueue = new LinkedList<>(); + private AudioFrame tmpFrame = null; - private boolean useAudio = false; + private VideoOutputStreamContainer video_st = null; + private AudioOutputStreamContainer audio_st = null; + private avformat.AVOutputFormat fmt; + private avformat.AVFormatContext oc = new avformat.AVFormatContext(); - public FFMpegVideoEncoder(int width, int height, float frameRate, int sampleRate, String filename, boolean useAudio){ + private Queue imageQueue = new LinkedList<>(); + private Queue audioQueue = new LinkedList<>(); - this.useAudio = useAudio; + private boolean useAudio = false; - AVDictionary opt = new AVDictionary(); + public FFMpegVideoEncoder(int width, int height, float frameRate, int sampleRate, String filename, boolean useAudio) { + this.useAudio = useAudio; - /* allocate the output media context */ - avformat_alloc_output_context2(oc, null, null, filename); - if (oc.isNull()) { - LOGGER.error("Could not deduce output format from file extension: using MPEG."); - avformat_alloc_output_context2(oc, null, "mpeg", filename); - } - if (oc.isNull()){ - return; - } + AVDictionary opt = new AVDictionary(); - fmt = oc.oformat(); - if (fmt.video_codec() != AV_CODEC_ID_NONE) { - video_st = new VideoOutputStreamContainer(width, height, 2_000_000, frameRate, oc, fmt.video_codec(), opt); - } - - if (fmt.audio_codec() != AV_CODEC_ID_NONE && useAudio) { - audio_st = new AudioOutputStreamContainer(oc, fmt.audio_codec(), sampleRate, 128_000, opt); - } + /* allocate the output media context */ + avformat_alloc_output_context2(oc, null, null, filename); + if (oc.isNull()) { + LOGGER.error("Could not deduce output format from file extension: using MPEG."); + avformat_alloc_output_context2(oc, null, "mpeg", filename); + } + if (oc.isNull()) { + return; + } - av_dump_format(oc, 0, filename, 1); + fmt = oc.oformat(); - int ret; + if (fmt.video_codec() != AV_CODEC_ID_NONE) { + video_st = new VideoOutputStreamContainer(width, height, 2_000_000, frameRate, oc, fmt.video_codec(), opt); + } - /* open the output file, if needed */ - if ((fmt.flags() & AVFMT_NOFILE) == 0) { - AVIOContext pb = new AVIOContext(null); - ret = avio_open(pb, filename, AVIO_FLAG_WRITE); - oc.pb(pb); - if (ret < 0) { - LOGGER.error("Could not open '{}': {}", filename, ret); - return; - } - } + if (fmt.audio_codec() != AV_CODEC_ID_NONE && useAudio) { + audio_st = new AudioOutputStreamContainer(oc, fmt.audio_codec(), sampleRate, 128_000, opt); + } - /* Write the stream header, if any. */ - ret = avformat_write_header(oc, opt); - if (ret < 0) { - LOGGER.error("Error occurred when opening output file: {}", ret); - } + av_dump_format(oc, 0, filename, 1); - } + int ret; - private void encode(){ - - while (!this.imageQueue.isEmpty() || !this.audioQueue.isEmpty()) { - boolean writtenPacket = false; - /* select the stream to encode */ - if (!useAudio || (av_compare_ts(video_st.frameCounter, video_st.c.time_base(), audio_st.next_pts, audio_st.c.time_base()) <= 0)) { - if (!this.imageQueue.isEmpty()){ - video_st.addFrame(this.imageQueue.poll()); - writtenPacket = true; - } - } else { - if (!this.audioQueue.isEmpty()){ - audio_st.addFrame(this.audioQueue.poll()); - writtenPacket = true; - } - } - - if (!writtenPacket){ - break; - } - } + /* open the output file, if needed */ + if ((fmt.flags() & AVFMT_NOFILE) == 0) { + AVIOContext pb = new AVIOContext(null); + ret = avio_open(pb, filename, AVIO_FLAG_WRITE); + oc.pb(pb); + if (ret < 0) { + LOGGER.error("Could not open '{}': {}", filename, ret); + return; + } } - public void add(MultiImage img){ - this.imageQueue.add(img); - encode(); + /* Write the stream header, if any. */ + ret = avformat_write_header(oc, opt); + if (ret < 0) { + LOGGER.error("Error occurred when opening output file: {}", ret); } - public void add(AudioFrame frame){ + } - int samples = audio_st.tmp_frame.nb_samples(); + private void encode() { - if (tmpFrame == null){ - tmpFrame = new AudioFrame(frame); - } else { - tmpFrame.append(frame); + while (!this.imageQueue.isEmpty() || !this.audioQueue.isEmpty()) { + boolean writtenPacket = false; + /* select the stream to encode */ + if (!useAudio || (av_compare_ts(video_st.frameCounter, video_st.c.time_base(), audio_st.next_pts, audio_st.c.time_base()) <= 0)) { + if (!this.imageQueue.isEmpty()) { + video_st.addFrame(this.imageQueue.poll()); + writtenPacket = true; } - while (tmpFrame.numberOfSamples() > samples){ - this.audioQueue.add(tmpFrame.split(samples)); + } else { + if (!this.audioQueue.isEmpty()) { + audio_st.addFrame(this.audioQueue.poll()); + writtenPacket = true; } + } - encode(); + if (!writtenPacket) { + break; + } } + } - public void add(VideoFrame frame){ - this.imageQueue.add(frame.getImage()); - if(frame.getAudio().isPresent()){ - this.add(frame.getAudio().get()); - } - encode(); + public void add(MultiImage img) { + this.imageQueue.add(img); + encode(); + } + + public void add(AudioFrame frame) { + + int samples = audio_st.tmp_frame.nb_samples(); + + if (tmpFrame == null) { + tmpFrame = new AudioFrame(frame); + } else { + tmpFrame.append(frame); + } + while (tmpFrame.numberOfSamples() > samples) { + this.audioQueue.add(tmpFrame.split(samples)); } - public void close(){ + encode(); + } - encode(); + public void add(VideoFrame frame) { + this.imageQueue.add(frame.getImage()); + if (frame.getAudio().isPresent()) { + this.add(frame.getAudio().get()); + } + encode(); + } - av_write_trailer(oc); + public void close() { - /* Close each codec. */ - if (video_st != null){ - video_st.close(); - } - if (audio_st != null){ - audio_st.close(); - } + encode(); - if ((fmt.flags() & AVFMT_NOFILE) == 0) - /* Close the output file. */ - avio_closep(oc.pb()); + av_write_trailer(oc); - /* free the stream */ - avformat_free_context(oc); + /* Close each codec. */ + if (video_st != null) { + video_st.close(); + } + if (audio_st != null) { + audio_st.close(); + } + if ((fmt.flags() & AVFMT_NOFILE) == 0) + /* Close the output file. */ { + avio_closep(oc.pb()); } + /* free the stream */ + avformat_free_context(oc); + + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/VideoOutputStreamContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/VideoOutputStreamContainer.java index e7610d9f7..ab8ce4688 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/VideoOutputStreamContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/VideoOutputStreamContainer.java @@ -1,161 +1,171 @@ package org.vitrivr.cineast.core.extraction.decode.video; +import static org.bytedeco.javacpp.avcodec.avcodec_free_context; +import static org.bytedeco.javacpp.avcodec.avcodec_open2; +import static org.bytedeco.javacpp.avcodec.avcodec_parameters_from_context; +import static org.bytedeco.javacpp.avutil.AVDictionary; +import static org.bytedeco.javacpp.avutil.AVFrame; +import static org.bytedeco.javacpp.avutil.AVRational; +import static org.bytedeco.javacpp.avutil.av_dict_copy; +import static org.bytedeco.javacpp.avutil.av_dict_free; +import static org.bytedeco.javacpp.avutil.av_frame_free; +import static org.bytedeco.javacpp.swscale.sws_freeContext; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.avcodec; +import org.bytedeco.javacpp.avformat; +import org.bytedeco.javacpp.avutil; +import org.bytedeco.javacpp.swscale; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.util.MathHelper; -import static org.bytedeco.javacpp.avcodec.*; -import static org.bytedeco.javacpp.avutil.*; -import static org.bytedeco.javacpp.swscale.sws_freeContext; - class VideoOutputStreamContainer extends AbstractAVStreamContainer { - private static final Logger LOGGER = LogManager.getLogger(); - int frameCounter = 0; - private AVFrame rgbFrame, outFrame; - private swscale.SwsContext sws_ctx; + private static final Logger LOGGER = LogManager.getLogger(); + int frameCounter = 0; + private AVFrame rgbFrame, outFrame; + private swscale.SwsContext sws_ctx; + + VideoOutputStreamContainer(int width, int height, int bitRate, float frameRate, avformat.AVFormatContext oc, int codec_id, AVDictionary opt) { + super(oc, codec_id); + + if (codec.type() != avutil.AVMEDIA_TYPE_VIDEO) { + LOGGER.error("Not a video codec"); + return; + } + + int[] frameRateFraction = MathHelper.toFraction(frameRate); - VideoOutputStreamContainer(int width, int height, int bitRate, float frameRate, avformat.AVFormatContext oc, int codec_id, AVDictionary opt) { - super(oc, codec_id); + c.bit_rate(bitRate); + c.width(width); + c.height(height); - if (codec.type() != avutil.AVMEDIA_TYPE_VIDEO) { - LOGGER.error("Not a video codec"); - return; - } + c.gop_size(10); + c.max_b_frames(1); + c.pix_fmt(avutil.AV_PIX_FMT_YUV420P); - int[] frameRateFraction = MathHelper.toFraction(frameRate); - - c.bit_rate(bitRate); - c.width(width); - c.height(height); - - c.gop_size(10); - c.max_b_frames(1); - c.pix_fmt(avutil.AV_PIX_FMT_YUV420P); - - AVRational timeBase = new AVRational(); - timeBase.num(frameRateFraction[1]); - timeBase.den(frameRateFraction[0]); - st.time_base(timeBase); - c.time_base(st.time_base()); + AVRational timeBase = new AVRational(); + timeBase.num(frameRateFraction[1]); + timeBase.den(frameRateFraction[0]); + st.time_base(timeBase); + c.time_base(st.time_base()); - AVRational fps = new AVRational(); - fps.den(frameRateFraction[1]); - fps.num(frameRateFraction[0]); - c.framerate(fps); + AVRational fps = new AVRational(); + fps.den(frameRateFraction[1]); + fps.num(frameRateFraction[0]); + c.framerate(fps); - if (c.codec_id() == avcodec.AV_CODEC_ID_MPEG2VIDEO) { - c.max_b_frames(2); - } - if (c.codec_id() == avcodec.AV_CODEC_ID_MPEG1VIDEO) { - c.mb_decision(2); - } - if (codec.id() == avcodec.AV_CODEC_ID_H264) { - avutil.av_opt_set(c.priv_data(), "preset", "slow", 0); - } - - - if ((oc.oformat().flags() & avformat.AVFMT_GLOBALHEADER) != 0) { - oc.oformat().flags(oc.oformat().flags() | avformat.AVFMT_GLOBALHEADER); - } - - AVDictionary topt = new AVDictionary(); - - av_dict_copy(topt, opt, 0); - - /* open the codec */ - int ret = avcodec_open2(c, codec, topt); - av_dict_free(topt); - if (ret < 0) { - LOGGER.error("Could not open video codec: {}", ret); - return; - } - - rgbFrame = avutil.av_frame_alloc(); - if (rgbFrame == null) { - LOGGER.error("Could not allocate frame"); - return; - } - - rgbFrame.format(avutil.AV_PIX_FMT_RGB24); - rgbFrame.width(c.width()); - rgbFrame.height(c.height()); - - ret = avutil.av_frame_get_buffer(rgbFrame, 32); - if (ret < 0) { - LOGGER.error("Could not allocate video frame data"); - return; - } - - outFrame = avutil.av_frame_alloc(); - if (outFrame == null) { - LOGGER.error("Could not allocate frame"); - return; - } - - outFrame.format(c.pix_fmt()); - outFrame.width(c.width()); - outFrame.height(c.height()); - - ret = avutil.av_frame_get_buffer(outFrame, 32); - if (ret < 0) { - LOGGER.error("Could not allocate video frame data"); - return; - } - - /* copy the stream parameters to the muxer */ - ret = avcodec_parameters_from_context(st.codecpar(), c); - if (ret < 0) { - LOGGER.error("Could not copy the stream parameters"); - return; - } - - sws_ctx = swscale.sws_getContext(c.width(), c.height(), avutil.AV_PIX_FMT_RGB24, c.width(), c.height(), c.pix_fmt(), swscale.SWS_BILINEAR, null, null, (DoublePointer) null); - - } - - - void addFrame(MultiImage img) { - - int ret = avutil.av_frame_make_writable(outFrame); - if (ret < 0) { - return; - } - - ret = avutil.av_frame_make_writable(rgbFrame); - if (ret < 0) { - return; - } - - int[] pixels = img.getColors(); - for (int i = 0; i < pixels.length; ++i) { - rgbFrame.data(0).put(3 * i, (byte) (((pixels[i]) >> 16) & 0xff)); - rgbFrame.data(0).put(3 * i + 1, (byte) (((pixels[i]) >> 8) & 0xff)); - rgbFrame.data(0).put(3 * i + 2, (byte) ((pixels[i]) & 0xff)); - } - - swscale.sws_scale(sws_ctx, rgbFrame.data(), rgbFrame.linesize(), 0, outFrame.height(), outFrame.data(), outFrame.linesize()); - - outFrame.pts(this.frameCounter++); - encode(c, outFrame, pkt); - } - - void close() { - if (c != null && !c.isNull()) { - avcodec_free_context(c); - } - if (rgbFrame != null && !rgbFrame.isNull()) { - av_frame_free(rgbFrame); - } - if (outFrame != null && !outFrame.isNull()) { - av_frame_free(outFrame); - } - if (sws_ctx != null && !sws_ctx.isNull()) { - sws_freeContext(sws_ctx); - } + if (c.codec_id() == avcodec.AV_CODEC_ID_MPEG2VIDEO) { + c.max_b_frames(2); + } + if (c.codec_id() == avcodec.AV_CODEC_ID_MPEG1VIDEO) { + c.mb_decision(2); + } + if (codec.id() == avcodec.AV_CODEC_ID_H264) { + avutil.av_opt_set(c.priv_data(), "preset", "slow", 0); + } + if ((oc.oformat().flags() & avformat.AVFMT_GLOBALHEADER) != 0) { + oc.oformat().flags(oc.oformat().flags() | avformat.AVFMT_GLOBALHEADER); } + AVDictionary topt = new AVDictionary(); + + av_dict_copy(topt, opt, 0); + + /* open the codec */ + int ret = avcodec_open2(c, codec, topt); + av_dict_free(topt); + if (ret < 0) { + LOGGER.error("Could not open video codec: {}", ret); + return; + } + + rgbFrame = avutil.av_frame_alloc(); + if (rgbFrame == null) { + LOGGER.error("Could not allocate frame"); + return; + } + + rgbFrame.format(avutil.AV_PIX_FMT_RGB24); + rgbFrame.width(c.width()); + rgbFrame.height(c.height()); + + ret = avutil.av_frame_get_buffer(rgbFrame, 32); + if (ret < 0) { + LOGGER.error("Could not allocate video frame data"); + return; + } + + outFrame = avutil.av_frame_alloc(); + if (outFrame == null) { + LOGGER.error("Could not allocate frame"); + return; + } + + outFrame.format(c.pix_fmt()); + outFrame.width(c.width()); + outFrame.height(c.height()); + + ret = avutil.av_frame_get_buffer(outFrame, 32); + if (ret < 0) { + LOGGER.error("Could not allocate video frame data"); + return; + } + + /* copy the stream parameters to the muxer */ + ret = avcodec_parameters_from_context(st.codecpar(), c); + if (ret < 0) { + LOGGER.error("Could not copy the stream parameters"); + return; + } + + sws_ctx = swscale.sws_getContext(c.width(), c.height(), avutil.AV_PIX_FMT_RGB24, c.width(), c.height(), c.pix_fmt(), swscale.SWS_BILINEAR, null, null, (DoublePointer) null); + + } + + + void addFrame(MultiImage img) { + + int ret = avutil.av_frame_make_writable(outFrame); + if (ret < 0) { + return; + } + + ret = avutil.av_frame_make_writable(rgbFrame); + if (ret < 0) { + return; + } + + int[] pixels = img.getColors(); + for (int i = 0; i < pixels.length; ++i) { + rgbFrame.data(0).put(3 * i, (byte) (((pixels[i]) >> 16) & 0xff)); + rgbFrame.data(0).put(3 * i + 1, (byte) (((pixels[i]) >> 8) & 0xff)); + rgbFrame.data(0).put(3 * i + 2, (byte) ((pixels[i]) & 0xff)); + } + + swscale.sws_scale(sws_ctx, rgbFrame.data(), rgbFrame.linesize(), 0, outFrame.height(), outFrame.data(), outFrame.linesize()); + + outFrame.pts(this.frameCounter++); + encode(c, outFrame, pkt); + } + + void close() { + if (c != null && !c.isNull()) { + avcodec_free_context(c); + } + if (rgbFrame != null && !rgbFrame.isNull()) { + av_frame_free(rgbFrame); + } + if (outFrame != null && !outFrame.isNull()) { + av_frame_free(outFrame); + } + if (sws_ctx != null && !sws_ctx.isNull()) { + sws_freeContext(sws_ctx); + } + + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/FileNameObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/FileNameObjectIdGenerator.java index 82b6a8ea1..1c9e25fcf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/FileNameObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/FileNameObjectIdGenerator.java @@ -1,10 +1,9 @@ package org.vitrivr.cineast.core.extraction.idgenerator; -import org.apache.commons.io.FilenameUtils; -import org.vitrivr.cineast.core.data.MediaType; - import java.nio.file.Path; import java.util.Map; +import org.apache.commons.io.FilenameUtils; +import org.vitrivr.cineast.core.data.MediaType; /** * Generates object IDs from object file names (without file extension). diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/JSONProvidedObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/JSONProvidedObjectIdGenerator.java index d42af82ac..db3e294d7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/JSONProvidedObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/JSONProvidedObjectIdGenerator.java @@ -1,84 +1,89 @@ package org.vitrivr.cineast.core.extraction.idgenerator; -import org.vitrivr.cineast.core.data.MediaType; -import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; -import org.vitrivr.cineast.core.util.json.JsonReader; - import java.io.File; import java.nio.file.Path; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; +import org.vitrivr.cineast.core.data.MediaType; +import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; +import org.vitrivr.cineast.core.util.json.JsonReader; /** - * Generates objectIds from a provided list of objectId's. Those ID's are either assigned in sequence OR mapped - * based on the filename. It is up to the author of such a list to ensure that there are enough ID's for the - * files in a run AND that those ID's are unique. - * + * Generates objectIds from a provided list of objectId's. Those ID's are either assigned in sequence OR mapped based on the filename. It is up to the author of such a list to ensure that there are enough ID's for the files in a run AND that those ID's are unique. */ public class JSONProvidedObjectIdGenerator implements ObjectIdGenerator { - /** - * Defines the assignment-modes for provided objectIds. - */ - private enum AssignmentMode { - MAP, /* Expects a JSON object with {:} pairs as entries. Each path is mapped to its ID. */ - CONTINUOUS /* Expects a JSON array with one ID per row. The ID's are assigned in a continuous fashion. */ - } + /** + * Defines the assignment-modes for provided objectIds. + */ + private enum AssignmentMode { + MAP, /* Expects a JSON object with {:} pairs as entries. Each path is mapped to its ID. */ + CONTINUOUS /* Expects a JSON array with one ID per row. The ID's are assigned in a continuous fashion. */ + } - /** Property-name for a custom start value (can be set in the configuration). */ - private static final String PROPERTY_NAME_SOURCE = "source"; + /** + * Property-name for a custom start value (can be set in the configuration). + */ + private static final String PROPERTY_NAME_SOURCE = "source"; - /** Property-name for a custom format (can be set in the configuration). */ - private static final String PROPERTY_NAME_ASSIGNMENT = "assignment"; + /** + * Property-name for a custom format (can be set in the configuration). + */ + private static final String PROPERTY_NAME_ASSIGNMENT = "assignment"; - /** Map that maps filenames to ID's. Only used in MAP mode. */ - private final HashMap pathIdMap; + /** + * Map that maps filenames to ID's. Only used in MAP mode. + */ + private final HashMap pathIdMap; - /** List of ID's. Only used in CONTINUOUS mode. */ - private final LinkedList idList; + /** + * List of ID's. Only used in CONTINUOUS mode. + */ + private final LinkedList idList; - /** The mode of assignment for ID's. */ - private final AssignmentMode mode; + /** + * The mode of assignment for ID's. + */ + private final AssignmentMode mode; - /** - * Constructor for {@link JSONProvidedObjectIdGenerator}. - * - * @param properties HashMap of named parameters. The values 'source' and 'assignment' are supported parameter keys. - */ - @SuppressWarnings("unchecked") - public JSONProvidedObjectIdGenerator(Map properties) { - String assignment = properties.get(PROPERTY_NAME_ASSIGNMENT); - if (assignment != null) { - this.mode = AssignmentMode.valueOf(assignment.toUpperCase()); - } else { - this.mode = AssignmentMode.MAP; - } - final String source = properties.get(PROPERTY_NAME_SOURCE); - final JsonReader reader = new JacksonJsonProvider(); - if (mode == AssignmentMode.MAP) { - this.pathIdMap = reader.toObject(new File(source), HashMap.class); - this.idList = null; - } else { - this.idList = reader.toObject(new File(source), LinkedList.class); - this.pathIdMap = null; - } + /** + * Constructor for {@link JSONProvidedObjectIdGenerator}. + * + * @param properties HashMap of named parameters. The values 'source' and 'assignment' are supported parameter keys. + */ + @SuppressWarnings("unchecked") + public JSONProvidedObjectIdGenerator(Map properties) { + String assignment = properties.get(PROPERTY_NAME_ASSIGNMENT); + if (assignment != null) { + this.mode = AssignmentMode.valueOf(assignment.toUpperCase()); + } else { + this.mode = AssignmentMode.MAP; + } + final String source = properties.get(PROPERTY_NAME_SOURCE); + final JsonReader reader = new JacksonJsonProvider(); + if (mode == AssignmentMode.MAP) { + this.pathIdMap = reader.toObject(new File(source), HashMap.class); + this.idList = null; + } else { + this.idList = reader.toObject(new File(source), LinkedList.class); + this.pathIdMap = null; } + } - /** - * Generates the next objectId and returns it as a string. That objectId not have a - * MediaType prefix! - * - * @param path Path to the file for which an ID should be generated. - * @param type MediaType of the file for which an ID should be generated. - * @return Next ID in the sequence. - */ - @Override - public String next(Path path, MediaType type) { - if (mode == AssignmentMode.MAP) { - return this.pathIdMap.get(path.getFileName().toString()).toString(); - } else { - return this.idList.poll(); - } + /** + * Generates the next objectId and returns it as a string. That objectId not have a MediaType prefix! + * + * @param path Path to the file for which an ID should be generated. + * @param type MediaType of the file for which an ID should be generated. + * @return Next ID in the sequence. + */ + @Override + public String next(Path path, MediaType type) { + if (mode == AssignmentMode.MAP) { + return this.pathIdMap.get(path.getFileName().toString()).toString(); + } else { + return this.idList.poll(); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ObjectIdGenerator.java index 9339b81fc..9392b8642 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ObjectIdGenerator.java @@ -1,29 +1,23 @@ package org.vitrivr.cineast.core.extraction.idgenerator; -import org.vitrivr.cineast.core.data.MediaType; - import java.nio.file.Path; +import org.vitrivr.cineast.core.data.MediaType; /** - * Classes implementing this interface are intended to generate ID's for multimedia-objects. The classes should - * be designed in such a way that: - * - * - The same instance of the class can be re-used (e.g. for an extraction run). - * - Two invocations of next() generate different ID's, unless path & type are identical for both invocations AND the class - * intends to generate ID's specific for the combination of these values. - * + * Classes implementing this interface are intended to generate ID's for multimedia-objects. The classes should be designed in such a way that: + *

    + * - The same instance of the class can be re-used (e.g. for an extraction run). - Two invocations of next() generate different ID's, unless path & type are identical for both invocations AND the class intends to generate ID's specific for the combination of these values. */ public interface ObjectIdGenerator { - /** - * Generates the next objectId and returns it as a string. That objectId should - * already contain the MediaType prefix, if the ID type supports media-type prefixing. - * - * Important: If the supply of ID's is depleted OR no ID could be generated for some reason, - * this method returns null! - * - * @param path Path to the file for which an ID should be generated. - * @param type MediaType of the file for which an ID should be generated. - * @return Next ID in the sequence or null - */ - String next(Path path, MediaType type); + + /** + * Generates the next objectId and returns it as a string. That objectId should already contain the MediaType prefix, if the ID type supports media-type prefixing. + *

    + * Important: If the supply of ID's is depleted OR no ID could be generated for some reason, this method returns null! + * + * @param path Path to the file for which an ID should be generated. + * @param type MediaType of the file for which an ID should be generated. + * @return Next ID in the sequence or null + */ + String next(Path path, MediaType type); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java index ed763cb48..6f6f04578 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/SequentialObjectIdGenerator.java @@ -1,59 +1,68 @@ package org.vitrivr.cineast.core.extraction.idgenerator; -import org.vitrivr.cineast.core.data.MediaType; - import java.nio.file.Path; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import org.vitrivr.cineast.core.data.MediaType; /** * Generates objectIds from a counter that is incremented every time a new ID is generated. - * */ public class SequentialObjectIdGenerator implements ObjectIdGenerator { - /** Property-name for a custom start value (can be set in the configuration). */ - private static final String PROPERTY_START_KEY = "start"; - /** Property-name for a custom format (can be set in the configuration). */ - private static final String PROPERTY_FORMAT_KEY = "format"; + /** + * Property-name for a custom start value (can be set in the configuration). + */ + private static final String PROPERTY_START_KEY = "start"; - /** Default value for the format. */ - private static final String PROPERTY_FORMAT_DEFAULT = "%07d"; + /** + * Property-name for a custom format (can be set in the configuration). + */ + private static final String PROPERTY_FORMAT_KEY = "format"; - /** Internal counter used to keep track of the ID's. */ - private final AtomicLong counter = new AtomicLong(1); + /** + * Default value for the format. + */ + private static final String PROPERTY_FORMAT_DEFAULT = "%07d"; - /** String format used to generate a String representation of the incremental ID. */ - private final String format; + /** + * Internal counter used to keep track of the ID's. + */ + private final AtomicLong counter = new AtomicLong(1); - /** - * Constructor for {@link JSONProvidedObjectIdGenerator}. - */ - public SequentialObjectIdGenerator() { - this.format = PROPERTY_FORMAT_DEFAULT; - } + /** + * String format used to generate a String representation of the incremental ID. + */ + private final String format; - /** - * Constructor for {@link JSONProvidedObjectIdGenerator}. - * - * @param properties HashMap of named parameters. The values 'start' and 'format' are supported parameter keys. - */ - public SequentialObjectIdGenerator(Map properties) { - String start = properties.get(PROPERTY_START_KEY); - if (start != null) { - this.counter.set(Long.parseLong(start)); - } - this.format = properties.getOrDefault(PROPERTY_FORMAT_KEY, PROPERTY_FORMAT_DEFAULT); - } + /** + * Constructor for {@link JSONProvidedObjectIdGenerator}. + */ + public SequentialObjectIdGenerator() { + this.format = PROPERTY_FORMAT_DEFAULT; + } - /** - * Generates the next ID in the sequence. - * - * @param path Path to the file for which an ID should be generated. - * @param type MediaType of the file for which an ID should be generated. - */ - @Override - public String next(Path path, MediaType type) { - return MediaType.generateId(type, String.format(this.format,this.counter.getAndIncrement())); + /** + * Constructor for {@link JSONProvidedObjectIdGenerator}. + * + * @param properties HashMap of named parameters. The values 'start' and 'format' are supported parameter keys. + */ + public SequentialObjectIdGenerator(Map properties) { + String start = properties.get(PROPERTY_START_KEY); + if (start != null) { + this.counter.set(Long.parseLong(start)); } + this.format = properties.getOrDefault(PROPERTY_FORMAT_KEY, PROPERTY_FORMAT_DEFAULT); + } + + /** + * Generates the next ID in the sequence. + * + * @param path Path to the file for which an ID should be generated. + * @param type MediaType of the file for which an ID should be generated. + */ + @Override + public String next(Path path, MediaType type) { + return MediaType.generateId(type, String.format(this.format, this.counter.getAndIncrement())); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/Sha1ObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/Sha1ObjectIdGenerator.java index 99df0a39b..5a3e3ec53 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/Sha1ObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/Sha1ObjectIdGenerator.java @@ -1,16 +1,15 @@ package org.vitrivr.cineast.core.extraction.idgenerator; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.util.LogHelper; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; - public class Sha1ObjectIdGenerator implements ObjectIdGenerator { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java index f1f39e2c2..d3fcfd16f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/UniqueObjectIdGenerator.java @@ -1,57 +1,62 @@ package org.vitrivr.cineast.core.extraction.idgenerator; -import org.vitrivr.cineast.core.data.MediaType; -import org.vitrivr.cineast.core.util.RandomStringGenerator; - import java.nio.file.Path; import java.util.HashSet; import java.util.Map; +import org.vitrivr.cineast.core.data.MediaType; +import org.vitrivr.cineast.core.util.RandomStringGenerator; /** * Generates an objectId from a random string. - * */ public class UniqueObjectIdGenerator implements ObjectIdGenerator { - /** Property-name for a custom start value (can be set in the configuration). */ - private static final String PROPERTY_LENGTH_KEY = "length"; - - /** Property-name for a custom start value (can be set in the configuration). */ - private static final Integer PROPERTY_LENGTH_DEFAULT = 16; - - /** Length of the generated ID. */ - private int length = 16; - - private HashSet usedIds = new HashSet<>(); - - /** - * Constructor for {@link UniqueObjectIdGenerator}. - */ - public UniqueObjectIdGenerator() { - this.length = PROPERTY_LENGTH_DEFAULT; - } - - /** - * Constructor for {@link UniqueObjectIdGenerator}. - * - * @param properties HashMap of named parameters. The values 'start' and 'format' are supported parameter keys. - */ - public UniqueObjectIdGenerator(Map properties) { - this.length = Integer.parseInt(properties.getOrDefault(PROPERTY_LENGTH_KEY, PROPERTY_LENGTH_DEFAULT.toString())); - } - - /** - * Generates the next ID in the sequence. - * - * @param path Path to the file for which an ID should be generated. - * @param type MediaType of the file for which an ID should be generated. - */ - @Override - public String next(Path path, MediaType type) { - String rawId; - do { - rawId = RandomStringGenerator.generateRandomString(this.length); - } while(this.usedIds.contains(rawId)); - this.usedIds.add(rawId); - return MediaType.generateId(type, rawId); - } + + /** + * Property-name for a custom start value (can be set in the configuration). + */ + private static final String PROPERTY_LENGTH_KEY = "length"; + + /** + * Property-name for a custom start value (can be set in the configuration). + */ + private static final Integer PROPERTY_LENGTH_DEFAULT = 16; + + /** + * Length of the generated ID. + */ + private int length = 16; + + private HashSet usedIds = new HashSet<>(); + + /** + * Constructor for {@link UniqueObjectIdGenerator}. + */ + public UniqueObjectIdGenerator() { + this.length = PROPERTY_LENGTH_DEFAULT; + } + + /** + * Constructor for {@link UniqueObjectIdGenerator}. + * + * @param properties HashMap of named parameters. The values 'start' and 'format' are supported parameter keys. + */ + public UniqueObjectIdGenerator(Map properties) { + this.length = Integer.parseInt(properties.getOrDefault(PROPERTY_LENGTH_KEY, PROPERTY_LENGTH_DEFAULT.toString())); + } + + /** + * Generates the next ID in the sequence. + * + * @param path Path to the file for which an ID should be generated. + * @param type MediaType of the file for which an ID should be generated. + */ + @Override + public String next(Path path, MediaType type) { + String rawId; + do { + rawId = RandomStringGenerator.generateRandomString(this.length); + } while (this.usedIds.contains(rawId)); + this.usedIds.add(rawId); + return MediaType.generateId(type, rawId); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/EXIFMetadataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/EXIFMetadataExtractor.java index ee8a34700..3cb029a26 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/EXIFMetadataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/EXIFMetadataExtractor.java @@ -3,10 +3,6 @@ import com.drew.metadata.exif.ExifDirectoryBase; import com.drew.metadata.exif.ExifSubIFDDirectory; import com.google.common.collect.Maps; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; -import org.vitrivr.cineast.core.data.providers.primitive.NothingProvider; -import org.vitrivr.cineast.core.util.MetadataUtil; - import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; @@ -14,6 +10,9 @@ import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.NothingProvider; +import org.vitrivr.cineast.core.util.MetadataUtil; public class EXIFMetadataExtractor implements MetadataExtractor { @@ -39,13 +38,11 @@ public class EXIFMetadataExtractor implements MetadataExtractor { /** - * Extracts the metadata from the specified path and returns a List of - * MediaObjectMetadataDescriptor objects (one for each metadata entry). + * Extracts the metadata from the specified path and returns a List of MediaObjectMetadataDescriptor objects (one for each metadata entry). * * @param objectId ID of the multimedia object for which metadata will be generated. - * @param path Path to the file for which metadata should be extracted. - * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be - * returned! + * @param path Path to the file for which metadata should be extracted. + * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be returned! */ @Override public List extract(String objectId, Path path) { @@ -56,8 +53,8 @@ public List extract(String objectId, Path path) { } Set> set = Maps.transformValues(FIELDS, md::getObject).entrySet(); return set.stream().filter(e -> e.getValue() != null).map( - e -> MediaObjectMetadataDescriptor.of(objectId, this.domain(), e.getKey(), e.getValue())) - .filter(e -> ! (e.getValueProvider() instanceof NothingProvider)) + e -> MediaObjectMetadataDescriptor.of(objectId, this.domain(), e.getKey(), e.getValue())) + .filter(e -> !(e.getValueProvider() instanceof NothingProvider)) .collect(Collectors.toList()); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IIIFMetaDataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IIIFMetaDataExtractor.java index 2b7869fa1..6f37f92f1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IIIFMetaDataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IIIFMetaDataExtractor.java @@ -1,4 +1,3 @@ - package org.vitrivr.cineast.core.extraction.metadata; import java.io.File; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IPTCMetadataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IPTCMetadataExtractor.java index 5afe36bc7..cc590eaf9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IPTCMetadataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/IPTCMetadataExtractor.java @@ -1,35 +1,32 @@ package org.vitrivr.cineast.core.extraction.metadata; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; - import java.nio.file.Path; import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; public class IPTCMetadataExtractor implements MetadataExtractor { - /** - * Extracts the metadata from the specified path and returns a List of MediaObjectMetadataDescriptor objects - * (one for each metadata entry). - * - * @param objectId ID of the multimedia object for which metadata will be generated. - * @param path Path to the file for which metadata should be extracted. - * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be returned! - */ - @Override - public List extract(String objectId, Path path) { - return null; - } + /** + * Extracts the metadata from the specified path and returns a List of MediaObjectMetadataDescriptor objects (one for each metadata entry). + * + * @param objectId ID of the multimedia object for which metadata will be generated. + * @param path Path to the file for which metadata should be extracted. + * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be returned! + */ + @Override + public List extract(String objectId, Path path) { + return null; + } - /** - * Returns a name that helps to identify the metadata domain. E.g. EXIF for EXIF - * metadata or DC for Dublin Core. - * - * @return Name of the metadata domain for which this extractor returns metadata. - */ - @Override - public String domain() { - return null; - } + /** + * Returns a name that helps to identify the metadata domain. E.g. EXIF for EXIF metadata or DC for Dublin Core. + * + * @return Name of the metadata domain for which this extractor returns metadata. + */ + @Override + public String domain() { + return null; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/JsonMetaDataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/JsonMetaDataExtractor.java index 7e22e3344..ae0acdf87 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/JsonMetaDataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/JsonMetaDataExtractor.java @@ -1,14 +1,13 @@ package org.vitrivr.cineast.core.extraction.metadata; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; -import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; - import java.io.File; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; +import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; public class JsonMetaDataExtractor implements MetadataExtractor { @@ -19,32 +18,32 @@ public List extract(String objectId, Path path) { File file = path.toFile(); File parent = file.getParentFile(); File jsonFile = new File(parent, file.getName() + ".json"); - - if(!jsonFile.exists()){ - jsonFile = new File(parent, com.google.common.io.Files.getNameWithoutExtension(file.getName()) + ".json"); + + if (!jsonFile.exists()) { + jsonFile = new File(parent, com.google.common.io.Files.getNameWithoutExtension(file.getName()) + ".json"); } - - if(!jsonFile.exists()){ + + if (!jsonFile.exists()) { return new ArrayList<>(0); } - + @SuppressWarnings("unchecked") Map json = jsonProvider.toObject(jsonFile, Map.class); - - if(json == null || json.isEmpty()){ + + if (json == null || json.isEmpty()) { return new ArrayList<>(0); } - + ArrayList _return = new ArrayList<>(json.size()); - + Set keys = json.keySet(); - - for(String key : keys){ + + for (String key : keys) { _return.add( MediaObjectMetadataDescriptor.of(objectId, domain(), key, json.get(key)) - ); + ); } - + return _return; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataExtractor.java index 5c7dab2eb..6d70f31ee 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataExtractor.java @@ -1,12 +1,12 @@ package org.vitrivr.cineast.core.extraction.metadata; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; - import java.nio.file.Path; import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; public interface MetadataExtractor { + /** * Initializes the extractor. The default implementation does nothing. */ @@ -15,13 +15,11 @@ default void init() { } /** - * Extracts the metadata from the specified path and returns a List of - * MediaObjectMetadataDescriptor objects (one for each metadata entry). + * Extracts the metadata from the specified path and returns a List of MediaObjectMetadataDescriptor objects (one for each metadata entry). * * @param objectId ID of the multimedia object for which metadata will be generated. - * @param path Path to the file for which metadata should be extracted. - * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be - * returned! + * @param path Path to the file for which metadata should be extracted. + * @return List of MultimediaMetadataDescriptors. The list may be empty but must always be returned! */ List extract(String objectId, Path path); @@ -33,8 +31,7 @@ default void finish() { } /** - * Returns a name that helps to identify the metadata domain. E.g. EXIF for EXIF - * metadata or DC for Dublin Core. + * Returns a name that helps to identify the metadata domain. E.g. EXIF for EXIF metadata or DC for Dublin Core. * * @return Name of the metadata domain for which this extractor returns metadata. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataFeatureExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataFeatureExtractor.java index b0919b859..a2aec6a34 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataFeatureExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/MetadataFeatureExtractor.java @@ -1,11 +1,10 @@ package org.vitrivr.cineast.core.extraction.metadata; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; - import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Optional; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; /** * A {@link MetadataExtractor} that extracts only a specific feature. @@ -13,12 +12,12 @@ * @param type of the feature data */ public interface MetadataFeatureExtractor extends MetadataExtractor { + /** - * Provides a default implementation by extracting the feature data and creating descriptors of - * it. + * Provides a default implementation by extracting the feature data and creating descriptors of it. * * @param objectId ID of the multimedia object for which metadata will be generated. - * @param path Path to the file for which metadata should be extracted. + * @param path Path to the file for which metadata should be extracted. * @return list of descriptors describing the feature data, if found, otherwise an empty list. */ @Override @@ -29,11 +28,12 @@ default List extract(String objectId, Path path) } /** - * Returns an {@link Optional} containing the extracted feature data from the file, if found, - * otherwise an empty {@code Optional}. + * Returns an {@link Optional} containing the extracted feature data from the file, if found, otherwise an empty {@code Optional}. */ Optional extractFeature(String objectId, Path path); - /** Returns a list of descriptors of the given feature data. */ + /** + * Returns a list of descriptors of the given feature data. + */ List createDescriptors(String objectId, T feature); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/TechnicalVideoMetadataExtractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/TechnicalVideoMetadataExtractor.java index 4c1be46c1..3aeecfded 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/TechnicalVideoMetadataExtractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/metadata/TechnicalVideoMetadataExtractor.java @@ -1,20 +1,23 @@ package org.vitrivr.cineast.core.extraction.metadata; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bytedeco.javacpp.*; -import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; -import org.vitrivr.cineast.core.extraction.decode.video.FFMpegVideoDecoder; -import org.vitrivr.cineast.core.util.MimeTypeHelper; +import static org.bytedeco.javacpp.avcodec.AVCodec; +import static org.bytedeco.javacpp.avformat.AVFormatContext; +import static org.bytedeco.javacpp.avformat.AVStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; - -import static org.bytedeco.javacpp.avcodec.AVCodec; -import static org.bytedeco.javacpp.avformat.AVFormatContext; -import static org.bytedeco.javacpp.avformat.AVStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.avcodec; +import org.bytedeco.javacpp.avformat; +import org.bytedeco.javacpp.avutil; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; +import org.vitrivr.cineast.core.extraction.decode.video.FFMpegVideoDecoder; +import org.vitrivr.cineast.core.util.MimeTypeHelper; public class TechnicalVideoMetadataExtractor implements MetadataExtractor { @@ -44,7 +47,7 @@ public class TechnicalVideoMetadataExtractor implements MetadataExtractor { * Extracts the technical video metadata from the specified path and returns a List of {@link MediaObjectMetadataDescriptor} objects (one for each metadata entry). * * @param objectId ID of the multimedia object for which metadata will be generated. - * @param path Path to the file for which metadata should be extracted. + * @param path Path to the file for which metadata should be extracted. * @return List of {@link MediaObjectMetadataDescriptor}s or an empty list, if extracting metadata fails. */ @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogram.java index 1599a4e82..91b3f147a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogram.java @@ -5,26 +5,26 @@ public class FuzzyColorHistogram extends Histogram { - public FuzzyColorHistogram() { - super(FuzzyColorHistogramQuantizer.Color.values().length); - int i = 0; - for(FuzzyColorHistogramQuantizer.Color c : FuzzyColorHistogramQuantizer.Color.values()){ - this.binNames.put(c.toString(), i++); - } - } - - @Override - public boolean areCompatible(Histogram hist) { - return hist instanceof FuzzyColorHistogram; - } - - public void add(FuzzyColorHistogramQuantizer.Color color){ - int index = this.binNames.get(color.toString()); - this.bins[index]++; - } - - public double getBin(FuzzyColorHistogramQuantizer.Color color){ - return getBin(color.toString()); - } + public FuzzyColorHistogram() { + super(FuzzyColorHistogramQuantizer.Color.values().length); + int i = 0; + for (FuzzyColorHistogramQuantizer.Color c : FuzzyColorHistogramQuantizer.Color.values()) { + this.binNames.put(c.toString(), i++); + } + } + + @Override + public boolean areCompatible(Histogram hist) { + return hist instanceof FuzzyColorHistogram; + } + + public void add(FuzzyColorHistogramQuantizer.Color color) { + int index = this.binNames.get(color.toString()); + this.bins[index]++; + } + + public double getBin(FuzzyColorHistogramQuantizer.Color color) { + return getBin(color.toString()); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogramCalculator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogramCalculator.java index 45369f59d..7a7e8f369 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogramCalculator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/FuzzyColorHistogramCalculator.java @@ -1,79 +1,79 @@ package org.vitrivr.cineast.core.extraction.segmenter; +import java.awt.image.BufferedImage; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.FuzzyColorHistogramQuantizer; import org.vitrivr.cineast.core.color.ReadableLabContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.data.frames.VideoFrame; -import java.awt.image.BufferedImage; - public class FuzzyColorHistogramCalculator { - private FuzzyColorHistogramCalculator(){} - - public static FuzzyColorHistogram getHistogram(VideoFrame f){ - return getHistogram(f.getImage().getBufferedImage()); - } - - public static FuzzyColorHistogram getHistogram(BufferedImage img){ - int[] colors = img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); - return getHistogram(colors); - } - - public static FuzzyColorHistogram getHistogram(int[] colors){ - FuzzyColorHistogram _return = new FuzzyColorHistogram(); - for(int color : colors){ - if(ReadableRGBContainer.getAlpha(color) < 127){ - continue; - } - ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(color); - FuzzyColorHistogramQuantizer.Color c = FuzzyColorHistogramQuantizer.quantize(lab); - _return.add(c); - } - return _return; - } - - public static FuzzyColorHistogram getHistogramNormalized(VideoFrame f){ - FuzzyColorHistogram _return = getHistogram(f); - _return.normalize(); - return _return; - } - - public static FuzzyColorHistogram getHistogramNormalized(BufferedImage img){ - FuzzyColorHistogram _return = getHistogram(img); - _return.normalize(); - return _return; - } - - public static SubdividedFuzzyColorHistogram getSubdividedHistogram(BufferedImage img, int subdivisions){ - int width = img.getWidth() / subdivisions, height = img.getHeight() / subdivisions; - SubdividedFuzzyColorHistogram hist = new SubdividedFuzzyColorHistogram(subdivisions); - for(int x = 0; x < subdivisions; ++x){ - for(int y = 0; y < subdivisions; ++y){ - int[] colors = img.getRGB(x * width, y * height, width, height, null, 0, width); - for(int color : colors){ - if(ReadableRGBContainer.getAlpha(color) < 127){ - continue; - } - ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(color); - FuzzyColorHistogramQuantizer.Color c = FuzzyColorHistogramQuantizer.quantize(lab); - hist.add(c, x * subdivisions + y); - } - } - } - return hist; - } - - public static SubdividedFuzzyColorHistogram getSubdividedHistogramNormalized(BufferedImage img, int subdivisions){ - SubdividedFuzzyColorHistogram hist = getSubdividedHistogram(img, subdivisions); - hist.normalize(); - return hist; - } + private FuzzyColorHistogramCalculator() { + } + + public static FuzzyColorHistogram getHistogram(VideoFrame f) { + return getHistogram(f.getImage().getBufferedImage()); + } + + public static FuzzyColorHistogram getHistogram(BufferedImage img) { + int[] colors = img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); + return getHistogram(colors); + } + + public static FuzzyColorHistogram getHistogram(int[] colors) { + FuzzyColorHistogram _return = new FuzzyColorHistogram(); + for (int color : colors) { + if (ReadableRGBContainer.getAlpha(color) < 127) { + continue; + } + ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(color); + FuzzyColorHistogramQuantizer.Color c = FuzzyColorHistogramQuantizer.quantize(lab); + _return.add(c); + } + return _return; + } + + public static FuzzyColorHistogram getHistogramNormalized(VideoFrame f) { + FuzzyColorHistogram _return = getHistogram(f); + _return.normalize(); + return _return; + } + + public static FuzzyColorHistogram getHistogramNormalized(BufferedImage img) { + FuzzyColorHistogram _return = getHistogram(img); + _return.normalize(); + return _return; + } + + public static SubdividedFuzzyColorHistogram getSubdividedHistogram(BufferedImage img, int subdivisions) { + int width = img.getWidth() / subdivisions, height = img.getHeight() / subdivisions; + SubdividedFuzzyColorHistogram hist = new SubdividedFuzzyColorHistogram(subdivisions); + for (int x = 0; x < subdivisions; ++x) { + for (int y = 0; y < subdivisions; ++y) { + int[] colors = img.getRGB(x * width, y * height, width, height, null, 0, width); + for (int color : colors) { + if (ReadableRGBContainer.getAlpha(color) < 127) { + continue; + } + ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(color); + FuzzyColorHistogramQuantizer.Color c = FuzzyColorHistogramQuantizer.quantize(lab); + hist.add(c, x * subdivisions + y); + } + } + } + return hist; + } + + public static SubdividedFuzzyColorHistogram getSubdividedHistogramNormalized(BufferedImage img, int subdivisions) { + SubdividedFuzzyColorHistogram hist = getSubdividedHistogram(img, subdivisions); + hist.normalize(); + return hist; + } - public static FuzzyColorHistogram getHistogramNormalized(int[] colors) { - FuzzyColorHistogram hist = getHistogram(colors); - hist.normalize(); - return hist; - } + public static FuzzyColorHistogram getHistogramNormalized(int[] colors) { + FuzzyColorHistogram hist = getHistogram(colors); + hist.normalize(); + return hist; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/SubdividedFuzzyColorHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/SubdividedFuzzyColorHistogram.java index ea9ff7581..269dde106 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/SubdividedFuzzyColorHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/SubdividedFuzzyColorHistogram.java @@ -5,32 +5,32 @@ public class SubdividedFuzzyColorHistogram extends Histogram { - private int subdivisions; - - public SubdividedFuzzyColorHistogram(int subdivisions) { - super(subdivisions * subdivisions * FuzzyColorHistogramQuantizer.Color.values().length); - this.subdivisions = subdivisions; - for(int s = 0; s < subdivisions*subdivisions; ++s){ - int i = 0; - for(FuzzyColorHistogramQuantizer.Color c : FuzzyColorHistogramQuantizer.Color.values()){ - this.binNames.put(c.toString() + s, i++); - } - } - } - - @Override - public boolean areCompatible(Histogram hist) { - return hist instanceof SubdividedFuzzyColorHistogram - && ((SubdividedFuzzyColorHistogram)hist).subdivisions == subdivisions; - } - - public void add(FuzzyColorHistogramQuantizer.Color color, int subdivision){ - int index = this.binNames.get(color.toString() + subdivision); - this.bins[index]++; - } - - public double getBin(FuzzyColorHistogramQuantizer.Color color, int subdivision){ - return getBin(color.toString() + subdivision); - } + private int subdivisions; + + public SubdividedFuzzyColorHistogram(int subdivisions) { + super(subdivisions * subdivisions * FuzzyColorHistogramQuantizer.Color.values().length); + this.subdivisions = subdivisions; + for (int s = 0; s < subdivisions * subdivisions; ++s) { + int i = 0; + for (FuzzyColorHistogramQuantizer.Color c : FuzzyColorHistogramQuantizer.Color.values()) { + this.binNames.put(c.toString() + s, i++); + } + } + } + + @Override + public boolean areCompatible(Histogram hist) { + return hist instanceof SubdividedFuzzyColorHistogram + && ((SubdividedFuzzyColorHistogram) hist).subdivisions == subdivisions; + } + + public void add(FuzzyColorHistogramQuantizer.Color color, int subdivision) { + int index = this.binNames.get(color.toString() + subdivision); + this.bins[index]++; + } + + public double getBin(FuzzyColorHistogramQuantizer.Color color, int subdivision) { + return getBin(color.toString() + subdivision); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java index fbb69d127..c36b16587 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/audio/ConstantLengthAudioSegmenter.java @@ -1,5 +1,10 @@ package org.vitrivr.cineast.core.extraction.segmenter.audio; +import java.util.ArrayDeque; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.data.segments.AudioSegment; @@ -9,200 +14,206 @@ import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.ArrayDeque; -import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - /** - * Merges multiple AudioFrames into a single AudioSegment using a constant number of frames per AudioSegment. The length - * of an AudioSegment in frames AND the overlapfactor between two subsequent AudioSegments can be defined upon onstruction of the - * ConstantLengthAudioSegmenter. + * Merges multiple AudioFrames into a single AudioSegment using a constant number of frames per AudioSegment. The length of an AudioSegment in frames AND the overlapfactor between two subsequent AudioSegments can be defined upon onstruction of the ConstantLengthAudioSegmenter. * * @see AudioSegment * @see AudioFrame - * */ public class ConstantLengthAudioSegmenter implements Segmenter { - /** Key in the configuration map used to configure the length setting. */ - private static final String PROPERTY_LENGTH_KEY = "length"; - - /** Key in the configuration map used to configure the overlap setting. */ - private static final String PROPERTY_OVERLAP_KEY = "overlap"; - - /** Key in the configuration map used to configure the length setting. */ - private static final Float PROPERTY_LENGTH_DEFAULT = 10.0f; - - /** Key in the configuration map used to configure the overlap setting. */ - private static final Float PROPERTY_OVERLAP_DEFAULT = 1.0f; - - private static final int SEGMENT_QUEUE_LENGTH = 10; - private static final int SEGMENT_POLLING_TIMEOUT = 1000; - - /** Decoder used for file decoding. */ - private Decoder decoder; - - /** A LinkedBlockingQueue used that holds the resulting AudioSegments. */ - private final LinkedBlockingQueue outputQueue = new LinkedBlockingQueue(SEGMENT_QUEUE_LENGTH); - - /** The length in AudioFrames of a resulting AudioSegment. */ - private final float length; - - /** Length of the overlap between two segments in seconds. */ - private final float overlap; - - /** AudioSegment that is currently filled with AudioFrames. */ - private AudioSegment currentSegment; - - /** ArrayDeque holding AudioFrames that have been queued for overlap. */ - private ArrayDeque overlapQueue = new ArrayDeque<>(); - - /** A flag indicating whether or not the segmenter has completed its work. */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Constructor for {@link ConstantLengthAudioSegmenter}. - * - * @param length Length of an individual segment in seconds. - * @param overlap Overlap between to subsequent AudioSegments - */ - public ConstantLengthAudioSegmenter(float length, float overlap) { - this.length = length; - this.overlap = overlap; - if (overlap >= 0.9f * length) { - throw new IllegalArgumentException("Overlap must be smaller than total segment length."); - } - } - - /** - * Constructor for {@link ConstantLengthAudioSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ConstantLengthAudioSegmenter} is created in. - */ - public ConstantLengthAudioSegmenter(ExtractionContextProvider context) { - this(PROPERTY_LENGTH_DEFAULT, PROPERTY_OVERLAP_DEFAULT); - } - - /** - * Constructor for {@link ConstantLengthAudioSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ConstantLengthAudioSegmenter} is created in. - * @param properties A HashMap containing the configuration properties for {@link ConstantLengthAudioSegmenter} - */ - public ConstantLengthAudioSegmenter(ExtractionContextProvider context, Map properties) { - this.length = Float.parseFloat(properties.getOrDefault(PROPERTY_LENGTH_KEY, PROPERTY_LENGTH_DEFAULT.toString())); - this.overlap = Float.parseFloat(properties.getOrDefault(PROPERTY_OVERLAP_KEY, PROPERTY_OVERLAP_DEFAULT.toString())); - if (overlap >= 0.9f * length) { - throw new IllegalArgumentException("Overlap must be smaller than total segment length."); - } + /** + * Key in the configuration map used to configure the length setting. + */ + private static final String PROPERTY_LENGTH_KEY = "length"; + + /** + * Key in the configuration map used to configure the overlap setting. + */ + private static final String PROPERTY_OVERLAP_KEY = "overlap"; + + /** + * Key in the configuration map used to configure the length setting. + */ + private static final Float PROPERTY_LENGTH_DEFAULT = 10.0f; + + /** + * Key in the configuration map used to configure the overlap setting. + */ + private static final Float PROPERTY_OVERLAP_DEFAULT = 1.0f; + + private static final int SEGMENT_QUEUE_LENGTH = 10; + private static final int SEGMENT_POLLING_TIMEOUT = 1000; + + /** + * Decoder used for file decoding. + */ + private Decoder decoder; + + /** + * A LinkedBlockingQueue used that holds the resulting AudioSegments. + */ + private final LinkedBlockingQueue outputQueue = new LinkedBlockingQueue(SEGMENT_QUEUE_LENGTH); + + /** + * The length in AudioFrames of a resulting AudioSegment. + */ + private final float length; + + /** + * Length of the overlap between two segments in seconds. + */ + private final float overlap; + + /** + * AudioSegment that is currently filled with AudioFrames. + */ + private AudioSegment currentSegment; + + /** + * ArrayDeque holding AudioFrames that have been queued for overlap. + */ + private ArrayDeque overlapQueue = new ArrayDeque<>(); + + /** + * A flag indicating whether or not the segmenter has completed its work. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Constructor for {@link ConstantLengthAudioSegmenter}. + * + * @param length Length of an individual segment in seconds. + * @param overlap Overlap between to subsequent AudioSegments + */ + public ConstantLengthAudioSegmenter(float length, float overlap) { + this.length = length; + this.overlap = overlap; + if (overlap >= 0.9f * length) { + throw new IllegalArgumentException("Overlap must be smaller than total segment length."); } - - /** - * Method used to initialize the Segmenter. A class implementing the Decoder interface with the same type must be provided. - * - * @param decoder Decoder used for frames-decoding. - * @param object Media object that is about to be segmented. - */ - @Override - public void init(Decoder decoder, MediaObjectDescriptor object) { - this.decoder = decoder; - this.complete.set(false); + } + + /** + * Constructor for {@link ConstantLengthAudioSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ConstantLengthAudioSegmenter} is created in. + */ + public ConstantLengthAudioSegmenter(ExtractionContextProvider context) { + this(PROPERTY_LENGTH_DEFAULT, PROPERTY_OVERLAP_DEFAULT); + } + + /** + * Constructor for {@link ConstantLengthAudioSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ConstantLengthAudioSegmenter} is created in. + * @param properties A HashMap containing the configuration properties for {@link ConstantLengthAudioSegmenter} + */ + public ConstantLengthAudioSegmenter(ExtractionContextProvider context, Map properties) { + this.length = Float.parseFloat(properties.getOrDefault(PROPERTY_LENGTH_KEY, PROPERTY_LENGTH_DEFAULT.toString())); + this.overlap = Float.parseFloat(properties.getOrDefault(PROPERTY_OVERLAP_KEY, PROPERTY_OVERLAP_DEFAULT.toString())); + if (overlap >= 0.9f * length) { + throw new IllegalArgumentException("Overlap must be smaller than total segment length."); } - - /** - * Returns the next SegmentContainer from the source OR null if there are no more segments in the outputQueue. As - * generation of SegmentContainers can take some time (depending on the media-type), a null return does not - * necessarily mean that the Segmenter is done segmenting. Use the complete() method to check this. - * - * Important: This method should be designed to block and wait for an appropriate amount of time if the Segmenter - * is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. - */ - @Override - public SegmentContainer getNext() throws InterruptedException { - SegmentContainer segment = this.outputQueue.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); - if (segment == null) { - this.complete.set(this.decoder.complete()); - } - return segment; + } + + /** + * Method used to initialize the Segmenter. A class implementing the Decoder interface with the same type must be provided. + * + * @param decoder Decoder used for frames-decoding. + * @param object Media object that is about to be segmented. + */ + @Override + public void init(Decoder decoder, MediaObjectDescriptor object) { + this.decoder = decoder; + this.complete.set(false); + } + + /** + * Returns the next SegmentContainer from the source OR null if there are no more segments in the outputQueue. As generation of SegmentContainers can take some time (depending on the media-type), a null return does not necessarily mean that the Segmenter is done segmenting. Use the complete() method to check this. + *

    + * Important: This method should be designed to block and wait for an appropriate amount of time if the Segmenter is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. + */ + @Override + public SegmentContainer getNext() throws InterruptedException { + SegmentContainer segment = this.outputQueue.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); + if (segment == null) { + this.complete.set(this.decoder.complete()); } - - /** - * Indicates whether the Segmenter is complete i.e. no new segments are to be expected. - * - * @return true if work is complete, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); + return segment; + } + + /** + * Indicates whether the Segmenter is complete i.e. no new segments are to be expected. + * + * @return true if work is complete, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Closes the Segmenter. This method should cleanup and relinquish all resources. Especially, calling this method should also close the Decoder associated with this Segmenter instance. + *

    + * Note: It is unsafe to re-use a Segmenter after it has been closed. + */ + @Override + public void close() { + if (this.complete.get()) { + if (this.decoder != null) { + this.decoder.close(); + } } - - /** - * Closes the Segmenter. This method should cleanup and relinquish all resources. Especially, - * calling this method should also close the Decoder associated with this Segmenter instance. - *

    - * Note: It is unsafe to re-use a Segmenter after it has been closed. - */ - @Override - public void close() { - if (this.complete.get()) { - if(this.decoder != null){ - this.decoder.close(); - } + } + + /** + * This methods pulls AudioFrames as they become available from the Decoder and adds them the the Deque of AudioFrames. Once that queue holds enough AudioFrames it is drained and a AudioSegment is emmitted. + *

    + * The overlapfactor defines how many frames re-enter the queue once they have been added to the new Segment. + * + * @see Thread#run() + */ + @Override + public void run() { + + this.currentSegment = new AudioSegment(); + + while (!this.decoder.complete()) { + AudioFrame newFrame = this.decoder.getNext(); + if (newFrame != null) { + this.currentSegment.addFrame(newFrame); + if (this.currentSegment.getAudioDuration() >= (this.length - this.overlap)) { + this.overlapQueue.offerLast(newFrame); } - } - - /** - * This methods pulls AudioFrames as they become available from the Decoder and adds them - * the the Deque of AudioFrames. Once that queue holds enough AudioFrames it is drained and - * a AudioSegment is emmitted. - * - * The overlapfactor defines how many frames re-enter the queue once they have been added to the - * new Segment. - * - * @see Thread#run() - */ - @Override - public void run() { - - this.currentSegment = new AudioSegment(); - - while (!this.decoder.complete()) { - AudioFrame newFrame = this.decoder.getNext(); - if (newFrame != null) { - this.currentSegment.addFrame(newFrame); - if (this.currentSegment.getAudioDuration() >= (this.length - this.overlap)) { - this.overlapQueue.offerLast(newFrame); - } - if (this.currentSegment.getAudioDuration() >= this.length) { - this.nextCycle(false); - } - } else if (this.decoder.complete()) { - this.nextCycle(true); - } + if (this.currentSegment.getAudioDuration() >= this.length) { + this.nextCycle(false); } + } else if (this.decoder.complete()) { + this.nextCycle(true); + } } + } - /** - * Drains the Deque and emits a new AudioSegment. - */ - private void nextCycle(boolean end) { - try { - this.outputQueue.put(this.currentSegment); - if (!end) { - this.currentSegment = new AudioSegment(); - AudioFrame frame; - while ((frame = this.overlapQueue.poll()) != null) { - this.currentSegment.addFrame(frame); - } - } else { - this.currentSegment = null; - this.overlapQueue.clear(); - } - } catch (InterruptedException e) { - e.printStackTrace(); + /** + * Drains the Deque and emits a new AudioSegment. + */ + private void nextCycle(boolean end) { + try { + this.outputQueue.put(this.currentSegment); + if (!end) { + this.currentSegment = new AudioSegment(); + AudioFrame frame; + while ((frame = this.overlapQueue.poll()) != null) { + this.currentSegment.addFrame(frame); } + } else { + this.currentSegment = null; + this.overlapQueue.clear(); + } + } catch (InterruptedException e) { + e.printStackTrace(); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/PassthroughSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/PassthroughSegmenter.java index 29432a11f..58991aa45 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/PassthroughSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/PassthroughSegmenter.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.extraction.segmenter.general; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -8,14 +10,10 @@ import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.extraction.segmenter.video.TRECVidMSRSegmenter; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.TimeUnit; - /** * A simple segmenter that passes the output from the decoder straight back to the orchestrator. - * + *

    * No aggregation or post-processing will take place besides wrapping of the content in a SegmentContainer. - * */ public abstract class PassthroughSegmenter implements Segmenter { @@ -49,7 +47,7 @@ public abstract class PassthroughSegmenter implements Segmenter { * Method used to initialize the {@link PassthroughSegmenter}. A class implementing the {@link Decoder} interface with the same type must be provided. * * @param decoder Decoder used for media-decoding. - * @param object Media object that is about to be segmented. + * @param object Media object that is about to be segmented. */ @Override public void init(Decoder decoder, MediaObjectDescriptor object) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java index a29dd4796..bf5c5a845 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/general/Segmenter.java @@ -5,45 +5,38 @@ import org.vitrivr.cineast.core.extraction.decode.general.Decoder; /** - * {@link Segmenter}s split a media file into chunks (segments). The nature of that chunk is specific to - * the media type and the{@link Segmenter}s implementation. A segment could be anything from a shot of a video - * to an arbitrary part of song or just a single image. - * + * {@link Segmenter}s split a media file into chunks (segments). The nature of that chunk is specific to the media type and the{@link Segmenter}s implementation. A segment could be anything from a shot of a video to an arbitrary part of song or just a single image. */ public interface Segmenter extends Runnable, AutoCloseable { - /** - * Method used to initialize the {@link Segmenter} and make it ready for use. It should be valid to call this multiple times - * in order to, for example, re-use the {@link Segmenter} with different {@link Decoder}s. In this case, this method is supposed to - * take care of the necessary cleanup, including but not limited to the disposal of the {@link Decoder}. - * - * @param decoder {@link Decoder} used for media decoding. - * @param object Media object that is about to be segmented. - */ - void init(Decoder decoder, MediaObjectDescriptor object); - /** - * Returns the next SegmentContainer from the source OR null if there are no more segments in the queue. As - * generation of SegmentContainers can take some time (depending on the media-type), a null return does not - * necessarily mean that the Segmenter is done segmenting. Use the complete() method to check this. - * - * Important: This method should be designed to block and wait for an appropriate amount of time if the - * Segmenter is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. - */ - SegmentContainer getNext() throws InterruptedException; + /** + * Method used to initialize the {@link Segmenter} and make it ready for use. It should be valid to call this multiple times in order to, for example, re-use the {@link Segmenter} with different {@link Decoder}s. In this case, this method is supposed to take care of the necessary cleanup, including but not limited to the disposal of the {@link Decoder}. + * + * @param decoder {@link Decoder} used for media decoding. + * @param object Media object that is about to be segmented. + */ + void init(Decoder decoder, MediaObjectDescriptor object); - /** - * Indicates that the {@link Segmenter} is complete i.e. no new segments are to be expected. - * - * @return true if work is complete, false otherwise. - */ - boolean complete(); + /** + * Returns the next SegmentContainer from the source OR null if there are no more segments in the queue. As generation of SegmentContainers can take some time (depending on the media-type), a null return does not necessarily mean that the Segmenter is done segmenting. Use the complete() method to check this. + * + * Important: This method should be designed to block and wait for an appropriate amount of time if the + * Segmenter is not yet ready to deliver another segment! It's up to the Segmenter how long that timeout should last. + */ + SegmentContainer getNext() throws InterruptedException; - /** - * Closes the {@link Segmenter}. This method should cleanup and relinquish all resources. Especially, - * calling this method should also close the {@link Decoder} associated with this {@link Segmenter} instance. - * - * Note: It is unsafe to re-use a {@link Segmenter} after it has been closed. - */ - @Override - void close(); + /** + * Indicates that the {@link Segmenter} is complete i.e. no new segments are to be expected. + * + * @return true if work is complete, false otherwise. + */ + boolean complete(); + + /** + * Closes the {@link Segmenter}. This method should cleanup and relinquish all resources. Especially, calling this method should also close the {@link Decoder} associated with this {@link Segmenter} instance. + * + * Note: It is unsafe to re-use a {@link Segmenter} after it has been closed. + */ + @Override + void close(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSegmenter.java index 73e719c66..39ac0a980 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSegmenter.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.extraction.segmenter.image; +import java.awt.image.BufferedImage; +import java.util.Map; import org.vitrivr.cineast.core.data.raw.CachedDataFactory; import org.vitrivr.cineast.core.data.segments.ImageSegment; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -8,49 +10,48 @@ import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.awt.image.BufferedImage; -import java.util.Map; - /** * A {@link Segmenter} that converts a {@link BufferedImage} into a {@link SegmentContainer}. * * @see Segmenter * @see PassthroughSegmenter - * */ public class ImageSegmenter extends PassthroughSegmenter { - /** THe {@link CachedDataFactory} that is used to create {@link org.vitrivr.cineast.core.data.raw.images.MultiImage}s. */ - private final CachedDataFactory factory; - /** - * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ImageSegmenter} is created in. - */ - public ImageSegmenter(ExtractionContextProvider context) { - super(); - this.factory = context.cacheConfig().sharedCachedDataFactory(); - } + /** + * THe {@link CachedDataFactory} that is used to create {@link org.vitrivr.cineast.core.data.raw.images.MultiImage}s. + */ + private final CachedDataFactory factory; + + /** + * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ImageSegmenter} is created in. + */ + public ImageSegmenter(ExtractionContextProvider context) { + super(); + this.factory = context.cacheConfig().sharedCachedDataFactory(); + } - /** - * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. - * @param properties A HashMap containing the configuration properties for {@link ImageSegmenter} - */ - public ImageSegmenter(ExtractionContextProvider context, Map properties) { - super(); - this.factory = context.cacheConfig().sharedCachedDataFactory(); - } + /** + * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. + * @param properties A HashMap containing the configuration properties for {@link ImageSegmenter} + */ + public ImageSegmenter(ExtractionContextProvider context, Map properties) { + super(); + this.factory = context.cacheConfig().sharedCachedDataFactory(); + } - /** - * Creates a new {@link SegmentContainer} from a {@link BufferedImage}. - * - * @param content The {@link BufferedImage} to extract a {@link SegmentContainer} from. - * @return {@link SegmentContainer} - */ - @Override - protected SegmentContainer getSegmentFromContent(BufferedImage content) { - return new ImageSegment(content, this.factory); - } + /** + * Creates a new {@link SegmentContainer} from a {@link BufferedImage}. + * + * @param content The {@link BufferedImage} to extract a {@link SegmentContainer} from. + * @return {@link SegmentContainer} + */ + @Override + protected SegmentContainer getSegmentFromContent(BufferedImage content) { + return new ImageSegment(content, this.factory); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java index 24f980a7e..86a8a3dda 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/image/ImageSequenceSegmenter.java @@ -1,8 +1,12 @@ package org.vitrivr.cineast.core.extraction.segmenter.image; +import java.awt.image.BufferedImage; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; @@ -15,48 +19,52 @@ import org.vitrivr.cineast.core.extraction.idgenerator.FileNameObjectIdGenerator; import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; -import java.awt.image.BufferedImage; - -import java.nio.file.Path; -import java.util.Optional; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; - /** * A {@link Segmenter} for {@link ImageSequence}s. - * */ public class ImageSequenceSegmenter implements Segmenter { - /** Logger facility. */ + /** + * Logger facility. + */ private static final Logger LOGGER = LogManager.getLogger(); - /** List of {@link BufferedImage}s currently processed.*/ + /** + * List of {@link BufferedImage}s currently processed. + */ private final Queue segments = new ConcurrentLinkedQueue<>(); - /** The {@link CachedDataFactory} that is used to create {@link org.vitrivr.cineast.core.data.raw.images.MultiImage}s. */ + /** + * The {@link CachedDataFactory} that is used to create {@link org.vitrivr.cineast.core.data.raw.images.MultiImage}s. + */ private final CachedDataFactory factory; /** * Instance of {@link FileNameObjectIdGenerator}. - * + *

    * Only Used in case that IDs should be derived from filenames which in the case of {@link MediaType#IMAGE_SEQUENCE} should also be done for the segments. */ private final FileNameObjectIdGenerator idgenerator; - /** {@link Decoder} instance used to decode {@link ImageSequence}. */ + /** + * {@link Decoder} instance used to decode {@link ImageSequence}. + */ private Decoder decoder = null; - /** Internal flag used to signal that segmentation is complete. */ + /** + * Internal flag used to signal that segmentation is complete. + */ private volatile boolean complete = false; - /** Internal flag used to signal that this {@link ImageSequenceSegmenter} is still running. */ + /** + * Internal flag used to signal that this {@link ImageSequenceSegmenter} is still running. + */ private volatile boolean running = false; - public ImageSequenceSegmenter(ExtractionContextProvider context){ + public ImageSequenceSegmenter(ExtractionContextProvider context) { this.factory = context.cacheConfig().sharedCachedDataFactory(); if (context.objectIdGenerator() instanceof FileNameObjectIdGenerator) { - this.idgenerator = (FileNameObjectIdGenerator)context.objectIdGenerator(); + this.idgenerator = (FileNameObjectIdGenerator) context.objectIdGenerator(); } else { this.idgenerator = null; } @@ -87,7 +95,7 @@ public synchronized boolean complete() { @Override public synchronized void close() { if (!this.running) { - if(this.decoder != null){ + if (this.decoder != null) { this.decoder.close(); } } @@ -103,7 +111,7 @@ public void run() { while (!this.decoder.complete()) { final ImageSequence sequence = this.decoder.getNext(); Pair> next; - while ((next = sequence.pop()) != null) + while ((next = sequence.pop()) != null) { if (next.second.isPresent()) { final ImageSegment segment = new ImageSegment(next.second.get(), this.factory); if (this.idgenerator != null) { @@ -111,6 +119,7 @@ public void run() { } this.segments.offer(segment); } + } } /* Sets the running flag to false. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/TRECVidMSRSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/TRECVidMSRSegmenter.java index 4557ba2a4..0b1a46003 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/TRECVidMSRSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/TRECVidMSRSegmenter.java @@ -1,5 +1,19 @@ package org.vitrivr.cineast.core.extraction.segmenter.video; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.Level; @@ -16,236 +30,249 @@ import org.vitrivr.cineast.core.extraction.segmenter.image.ImageSegmenter; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - public class TRECVidMSRSegmenter implements Segmenter { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** Maximum length of the output queue. */ - private static final int SEGMENT_QUEUE_LENGTH = 10; - - /** The timeout when polling the output queue. Defaults to 5s. */ - private static final int SEGMENT_POLLING_TIMEOUT = 5000; - - /** Key in the configuration map used to configure the MSR folder setting. */ - private static final String PROPERTY_FOLDER_KEY = "folder"; - - /** Queue of shot boundaries (pair of start and end frame). The entries are supposed to be sorted in ascending order. */ - private final Queue> boundaries = new ArrayDeque<>(); - - /** Queue for resulting {@link SegmentContainer}s waiting pick up by some consumer. */ - private final LinkedBlockingQueue outputQueue = new LinkedBlockingQueue<>(SEGMENT_QUEUE_LENGTH); - - /** Path to the folder relative to which MSR files will be looked up. */ - private final Path msrFolderPath; - - /** The {@link Decoder} instance used by the current instance of {@link TRECVidMSRSegmenter}*/ - private Decoder decoder; - - /** Internal flag that indicates whether the {@link TRECVidMSRSegmenter} is still running (= true). */ - private volatile boolean running = false; - - /** A flag indicating whether more {@link SegmentContainer}s are to be expected from this {@link TRECVidMSRSegmenter}. */ - private volatile boolean complete = false; - - /** - * Constructor for {@link TRECVidMSRSegmenter}. - * - * @param path Path to the folder relative to which MSR files will be resolved (based on the name of the input video file). - */ - public TRECVidMSRSegmenter(Path path) { - this.msrFolderPath = path; - if (!Files.isDirectory(path)) { - throw new IllegalArgumentException("The MSR path must point to a directory."); - } + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Maximum length of the output queue. + */ + private static final int SEGMENT_QUEUE_LENGTH = 10; + + /** + * The timeout when polling the output queue. Defaults to 5s. + */ + private static final int SEGMENT_POLLING_TIMEOUT = 5000; + + /** + * Key in the configuration map used to configure the MSR folder setting. + */ + private static final String PROPERTY_FOLDER_KEY = "folder"; + + /** + * Queue of shot boundaries (pair of start and end frame). The entries are supposed to be sorted in ascending order. + */ + private final Queue> boundaries = new ArrayDeque<>(); + + /** + * Queue for resulting {@link SegmentContainer}s waiting pick up by some consumer. + */ + private final LinkedBlockingQueue outputQueue = new LinkedBlockingQueue<>(SEGMENT_QUEUE_LENGTH); + + /** + * Path to the folder relative to which MSR files will be looked up. + */ + private final Path msrFolderPath; + + /** + * The {@link Decoder} instance used by the current instance of {@link TRECVidMSRSegmenter} + */ + private Decoder decoder; + + /** + * Internal flag that indicates whether the {@link TRECVidMSRSegmenter} is still running (= true). + */ + private volatile boolean running = false; + + /** + * A flag indicating whether more {@link SegmentContainer}s are to be expected from this {@link TRECVidMSRSegmenter}. + */ + private volatile boolean complete = false; + + /** + * Constructor for {@link TRECVidMSRSegmenter}. + * + * @param path Path to the folder relative to which MSR files will be resolved (based on the name of the input video file). + */ + public TRECVidMSRSegmenter(Path path) { + this.msrFolderPath = path; + if (!Files.isDirectory(path)) { + throw new IllegalArgumentException("The MSR path must point to a directory."); } - - /** - * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ImageSegmenter} is created in. - */ - public TRECVidMSRSegmenter(ExtractionContextProvider context) { - this(context.inputPath().orElse(null)); + } + + /** + * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ImageSegmenter} is created in. + */ + public TRECVidMSRSegmenter(ExtractionContextProvider context) { + this(context.inputPath().orElse(null)); + } + + /** + * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. + * + * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. + * @param properties A HashMap containing the configuration properties for {@link ImageSegmenter} + */ + public TRECVidMSRSegmenter(ExtractionContextProvider context, Map properties) { + this(properties.containsKey(PROPERTY_FOLDER_KEY) ? Paths.get(properties.get(PROPERTY_FOLDER_KEY)) : context.inputPath().orElse(null)); + } + + /** + * @param decoder Decoder used for media-decoding. + * @param object Media object that is about to be segmented. + */ + @Override + public synchronized void init(Decoder decoder, MediaObjectDescriptor object) { + if (!this.running) { + this.decoder = decoder; + this.complete = false; + this.boundaries.clear(); + + /* Loads the MSR file relative to the video. */ + final String suffix = object.getPath().substring(object.getPath().lastIndexOf("."), object.getPath().length()); + final String msrFilename = object.getPath().replace(suffix, ".msb"); + final Path path = this.msrFolderPath.resolve(msrFilename); + this.boundaries.addAll(decode(path)); + } else { + throw new IllegalArgumentException("You cannot call init() while the TRECVidMSRSegmenter is running."); } - - /** - * Constructor for {@link ImageSegmenter required for instantiation through {@link ReflectionHelper }. - * - * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. - * @param properties A HashMap containing the configuration properties for {@link ImageSegmenter} - */ - public TRECVidMSRSegmenter(ExtractionContextProvider context, Map properties) { - this(properties.containsKey(PROPERTY_FOLDER_KEY) ? Paths.get(properties.get(PROPERTY_FOLDER_KEY)) : context.inputPath().orElse(null)); + } + + /** + * Returns the next {@link SegmentContainer} or null, if there is currently no {@link SegmentContainer} available. + * + * @return {@link SegmentContainer} or null + * @throws InterruptedException If thread is interrupted while waiting fro the {@link SegmentContainer} + */ + @Override + public SegmentContainer getNext() throws InterruptedException { + SegmentContainer segment = this.outputQueue.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); + synchronized (this) { + if (segment == null && !this.running) { + this.complete = true; + } + } + return segment; + } + + /** + * Checks the current state of the {@link TRECVidMSRSegmenter}. The {@link TRECVidMSRSegmenter} is considered to be complete if the segmenting process has stopped AND the segments queue has been drained completely. + * + * @return Current state of the + */ + @Override + public synchronized boolean complete() { + return this.complete; + } + + /** + * Closes the {@link TRECVidMSRSegmenter} and all associated resources. This includes decoders. + */ + @Override + public synchronized void close() { + if (!this.running) { + this.decoder.close(); + } else { + throw new IllegalStateException("You cannot close the TRECVidMSRSegmenter while it is running."); + } + } + + /** + * This method takes care of the actual segmenting based on existing TRECVid master shot record boundaries. + */ + @Override + public void run() { + /* Sets the running flag to true. */ + synchronized (this) { + this.running = true; } - /** - * - * @param decoder Decoder used for media-decoding. - * @param object Media object that is about to be segmented. - */ - @Override - public synchronized void init(Decoder decoder, MediaObjectDescriptor object) { - if (!this.running) { - this.decoder = decoder; - this.complete = false; - this.boundaries.clear(); - - /* Loads the MSR file relative to the video. */ - final String suffix = object.getPath().substring(object.getPath().lastIndexOf("."), object.getPath().length()); - final String msrFilename = object.getPath().replace(suffix, ".msb"); - final Path path = this.msrFolderPath.resolve(msrFilename); - this.boundaries.addAll(decode(path)); - } else { - throw new IllegalArgumentException("You cannot call init() while the TRECVidMSRSegmenter is running."); + try { + VideoFrame currentFrame = null; + while (!this.boundaries.isEmpty()) { + final Pair boundary = this.boundaries.poll(); + if (boundary == null) { + break; } - } - /** - * Returns the next {@link SegmentContainer} or null, if there is currently no {@link SegmentContainer} available. - * - * @return {@link SegmentContainer} or null - * @throws InterruptedException If thread is interrupted while waiting fro the {@link SegmentContainer} - */ - @Override - public SegmentContainer getNext() throws InterruptedException { - SegmentContainer segment = this.outputQueue.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); - synchronized (this) { - if (segment == null && !this.running) { - this.complete = true; - } + final VideoSegment segment = new VideoSegment(); + + /* Append frames to the segment until the VideoFrame's (sequential) is beyond the boundaries. */ + while (!this.decoder.complete()) { + if (currentFrame == null) { + currentFrame = this.decoder.getNext(); + } + if (currentFrame.getId() >= boundary.getLeft() && currentFrame.getId() <= boundary.getRight()) { + segment.addVideoFrame(currentFrame); + currentFrame = null; + } else { + break; + } } - return segment; - } - /** - * Checks the current state of the {@link TRECVidMSRSegmenter}. The {@link TRECVidMSRSegmenter} is considered to - * be complete if the segmenting process has stopped AND the segments queue has been drained completely. - * - * @return Current state of the - */ - @Override - public synchronized boolean complete() { - return this.complete; + /* Add video segment to the segments queue. */ + this.outputQueue.put(segment); + } + + /* Create final segment. */ + final VideoSegment finalSegment = new VideoSegment(); + while (!this.decoder.complete()) { + finalSegment.addVideoFrame(this.decoder.getNext()); + } + if (finalSegment.getNumberOfFrames() > 0) { + this.outputQueue.put(finalSegment); + } + } catch (InterruptedException e) { + LOGGER.log(Level.ERROR, "The thread that runs the TRECVidMSRSegmenter was interrupted: {}", e); } - /** - * Closes the {@link TRECVidMSRSegmenter} and all associated resources. This includes decoders. - */ - @Override - public synchronized void close() { - if (!this.running) { - this.decoder.close(); - } else { - throw new IllegalStateException("You cannot close the TRECVidMSRSegmenter while it is running."); - } + /* Sets the running flag to false. */ + synchronized (this) { + this.running = false; } - - /** - * This method takes care of the actual segmenting based on existing TRECVid master shot record boundaries. - */ - @Override - public void run() { - /* Sets the running flag to true. */ - synchronized (this) { - this.running = true; + } + + /** + * Decodes shot boundaries in the format used for TRECVID and creates {@link MediaSegmentDescriptor}s accordingly. + * + * @param msr The file containing the master shot references. + */ + public static List> decode(Path msr) { + final List> _return = new ArrayList<>(); + try (final BufferedReader reader = Files.newBufferedReader(msr, StandardCharsets.ISO_8859_1)) { + String line = null; + int shotCounter = 0; + while ((line = reader.readLine()) != null) { + line = line.trim(); + + if (line.isEmpty()) { //skip empty lines + continue; } - try { - VideoFrame currentFrame = null; - while (!this.boundaries.isEmpty()) { - final Pair boundary = this.boundaries.poll(); - if (boundary == null) break; - - final VideoSegment segment = new VideoSegment(); - - /* Append frames to the segment until the VideoFrame's (sequential) is beyond the boundaries. */ - while (!this.decoder.complete()) { - if (currentFrame == null) currentFrame = this.decoder.getNext(); - if (currentFrame.getId() >= boundary.getLeft() && currentFrame.getId() <= boundary.getRight()) { - segment.addVideoFrame(currentFrame); - currentFrame = null; - } else { - break; - } - } - - /* Add video segment to the segments queue. */ - this.outputQueue.put(segment); - } - - /* Create final segment. */ - final VideoSegment finalSegment = new VideoSegment(); - while (!this.decoder.complete()) { - finalSegment.addVideoFrame(this.decoder.getNext()); - } - if (finalSegment.getNumberOfFrames() > 0) this.outputQueue.put(finalSegment); - } catch (InterruptedException e) { - LOGGER.log(Level.ERROR, "The thread that runs the TRECVidMSRSegmenter was interrupted: {}", e); + if (!Character.isDigit(line.charAt(0))) {//line does not start with a number + continue; } - /* Sets the running flag to false. */ - synchronized (this) { - this.running = false; + String[] split = line.split(" "); + if (split.length < 2) {//there are not two blocks on this line + continue; } - } - /** - * Decodes shot boundaries in the format used for TRECVID and creates {@link MediaSegmentDescriptor}s accordingly. - * - * @param msr The file containing the master shot references. - */ - public static List> decode(Path msr) { - final List> _return = new ArrayList<>(); - try (final BufferedReader reader = Files.newBufferedReader(msr, StandardCharsets.ISO_8859_1)) { - String line = null; - int shotCounter = 0; - while((line = reader.readLine()) != null){ - line = line.trim(); - - if(line.isEmpty()){ //skip empty lines - continue; - } - - if(!Character.isDigit(line.charAt(0))){//line does not start with a number - continue; - } - - String[] split = line.split(" "); - if(split.length < 2){//there are not two blocks on this line - continue; - } - - long start, end; - try{ - start = 1 + Long.parseLong(split[0]); //TRECVID msr starts with 0 - end = 1 + Long.parseLong(split[1]); - } catch(NumberFormatException e){ - continue; - } - - ++shotCounter; - - /* TODO: Derive absolute start and end position of MediaSegmentDescriptor. */ - _return.add(new ImmutablePair<>(start, end)); - } - } catch (FileNotFoundException e) { - LOGGER.error("TRECVid MSR file '{}' was not found.", msr.toString()); - } catch (IOException e) { - LOGGER.error("Error while reading RECVid MSR file '{}': {}", msr.toString(), e.getMessage()); + long start, end; + try { + start = 1 + Long.parseLong(split[0]); //TRECVID msr starts with 0 + end = 1 + Long.parseLong(split[1]); + } catch (NumberFormatException e) { + continue; } - return _return; + ++shotCounter; + + /* TODO: Derive absolute start and end position of MediaSegmentDescriptor. */ + _return.add(new ImmutablePair<>(start, end)); + } + } catch (FileNotFoundException e) { + LOGGER.error("TRECVid MSR file '{}' was not found.", msr.toString()); + } catch (IOException e) { + LOGGER.error("Error while reading RECVid MSR file '{}': {}", msr.toString(), e.getMessage()); } + + return _return; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/V3CMSBSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/V3CMSBSegmenter.java index 2732a1749..98c1dacad 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/V3CMSBSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/V3CMSBSegmenter.java @@ -1,5 +1,18 @@ package org.vitrivr.cineast.core.extraction.segmenter.video; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.Level; @@ -15,19 +28,11 @@ import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.extraction.segmenter.image.ImageSegmenter; -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - public class V3CMSBSegmenter implements Segmenter { - /** */ + /** + * + */ private static final Logger LOGGER = LogManager.getLogger(); /** @@ -46,8 +51,7 @@ public class V3CMSBSegmenter implements Segmenter { private static final String PROPERTY_FOLDER_KEY = "folder"; /** - * Queue of shot boundaries (pair of start and end frame). The entries are supposed to be sorted - * in ascending order. + * Queue of shot boundaries (pair of start and end frame). The entries are supposed to be sorted in ascending order. */ private final Queue> boundaries = new ArrayDeque<>(); @@ -73,16 +77,14 @@ public class V3CMSBSegmenter implements Segmenter { private volatile boolean running = false; /** - * A flag indicating whether more {@link SegmentContainer}s are to be expected from this {@link - * V3CMSBSegmenter}. + * A flag indicating whether more {@link SegmentContainer}s are to be expected from this {@link V3CMSBSegmenter}. */ private volatile boolean complete = false; /** * Constructor for {@link V3CMSBSegmenter}. * - * @param path Path to the folder relative to which MSR files will be resolved (based on the name - * of the input video file). + * @param path Path to the folder relative to which MSR files will be resolved (based on the name of the input video file). */ public V3CMSBSegmenter(Path path) { this.msrFolderPath = path; @@ -94,8 +96,7 @@ public V3CMSBSegmenter(Path path) { /** * Constructor for {@link ImageSegmenter}. * - * @param context The {@link ExtractionContextProvider } for the extraction context this {@link - * ImageSegmenter} is created in. + * @param context The {@link ExtractionContextProvider } for the extraction context this {@link ImageSegmenter} is created in. */ public V3CMSBSegmenter(ExtractionContextProvider context) { this(context.inputPath().orElse(null)); @@ -104,7 +105,7 @@ public V3CMSBSegmenter(ExtractionContextProvider context) { /** * Constructor for {@link ImageSegmenter}. * - * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. + * @param context The {@link ExtractionContextProvider} for the extraction context this {@link ImageSegmenter} is created in. * @param properties A HashMap containing the configuration properties for {@link ImageSegmenter} */ public V3CMSBSegmenter(ExtractionContextProvider context, Map properties) { @@ -114,8 +115,7 @@ public V3CMSBSegmenter(ExtractionContextProvider context, Map pr } /** - * Decodes shot boundaries in the format used for TRECVID and creates {@link MediaSegmentDescriptor}s - * accordingly. + * Decodes shot boundaries in the format used for TRECVID and creates {@link MediaSegmentDescriptor}s accordingly. * * @param msb The file containing the master shot references. */ @@ -163,7 +163,7 @@ public static List> decode(Path msb) { /** * @param decoder Decoder used for media-decoding. - * @param object Media object that is about to be segmented. + * @param object Media object that is about to be segmented. */ @Override public synchronized void init(Decoder decoder, MediaObjectDescriptor object) { @@ -185,12 +185,10 @@ public synchronized void init(Decoder decoder, MediaObjectDescriptor } /** - * Returns the next {@link SegmentContainer} or null, if there is currently no {@link - * SegmentContainer} available. + * Returns the next {@link SegmentContainer} or null, if there is currently no {@link SegmentContainer} available. * * @return {@link SegmentContainer} or null - * @throws InterruptedException If thread is interrupted while waiting fro the {@link - * SegmentContainer} + * @throws InterruptedException If thread is interrupted while waiting fro the {@link SegmentContainer} */ @Override public SegmentContainer getNext() throws InterruptedException { @@ -205,9 +203,7 @@ public SegmentContainer getNext() throws InterruptedException { } /** - * Checks the current state of the {@link V3CMSBSegmenter}. The {@link V3CMSBSegmenter} is - * considered to be complete if the segmenting process has stopped AND the segments queue has been - * drained completely. + * Checks the current state of the {@link V3CMSBSegmenter}. The {@link V3CMSBSegmenter} is considered to be complete if the segmenting process has stopped AND the segments queue has been drained completely. * * @return Current state of the */ @@ -229,8 +225,7 @@ public synchronized void close() { } /** - * This method takes care of the actual segmenting based on existing TRECVid master shot record - * boundaries. + * This method takes care of the actual segmenting based on existing TRECVid master shot record boundaries. */ @Override public void run() { @@ -253,7 +248,7 @@ public void run() { while (!this.decoder.complete()) { if (currentFrame == null) { currentFrame = this.decoder.getNext(); - if(currentFrame == null){ + if (currentFrame == null) { break; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java index c1c1bb793..45a692ff8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/segmenter/video/VideoHistogramSegmenter.java @@ -1,5 +1,12 @@ package org.vitrivr.cineast.core.extraction.segmenter.video; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,261 +24,257 @@ import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - public class VideoHistogramSegmenter implements Segmenter { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); - private static final double DEFAULT_THRESHOLD = 0.05; + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); - private static final int SEGMENT_QUEUE_LENGTH = 10; + private static final double DEFAULT_THRESHOLD = 0.05; - private static final int PRESHOT_QUEUE_LENGTH = 10; + private static final int SEGMENT_QUEUE_LENGTH = 10; - private static final int DEFAULT_MAX_SHOT_LENGTH = 720; + private static final int PRESHOT_QUEUE_LENGTH = 10; - private static final int SEGMENT_POLLING_TIMEOUT = 1000; + private static final int DEFAULT_MAX_SHOT_LENGTH = 720; - private final double threshold; + private static final int SEGMENT_POLLING_TIMEOUT = 1000; - private final int maxShotLength; + private final double threshold; - private Decoder decoder; + private final int maxShotLength; - private final LinkedList videoFrameList = new LinkedList<>(); + private Decoder decoder; - private final LinkedList> preShotList = new LinkedList<>(); + private final LinkedList videoFrameList = new LinkedList<>(); - private final LinkedBlockingQueue segments = new LinkedBlockingQueue<>(SEGMENT_QUEUE_LENGTH); + private final LinkedList> preShotList = new LinkedList<>(); - private final List knownShotBoundaries = new LinkedList<>(); + private final LinkedBlockingQueue segments = new LinkedBlockingQueue<>(SEGMENT_QUEUE_LENGTH); - private volatile boolean complete = false; + private final List knownShotBoundaries = new LinkedList<>(); - private volatile boolean isrunning = false; + private volatile boolean complete = false; - /** MediaSegmentReader used to lookup existing SegmentDescriptors during the extraction. */ - private final MediaSegmentReader segmentReader; + private volatile boolean isrunning = false; + /** + * MediaSegmentReader used to lookup existing SegmentDescriptors during the extraction. + */ + private final MediaSegmentReader segmentReader; - /** - * Constructor required for instantiates through {@link ReflectionHelper}. - */ - public VideoHistogramSegmenter(ExtractionContextProvider context) { - this(context, new HashMap<>(0)); - } - /** - * Constructor required for instantiates through {@link ReflectionHelper}. - */ - public VideoHistogramSegmenter(ExtractionContextProvider context, Map parameters) { - if (parameters.containsKey("threshold")) { - this.threshold = Double.parseDouble(parameters.get("threshold")); - } else { - this.threshold = DEFAULT_THRESHOLD; - } - if (parameters.containsKey("maxShotLength")) { - this.maxShotLength = Integer.parseInt(parameters.get("maxShotLength")); - } else { - this.maxShotLength = DEFAULT_MAX_SHOT_LENGTH; - } - this.segmentReader = new MediaSegmentReader(context.persistencyReader().get()); - } + /** + * Constructor required for instantiates through {@link ReflectionHelper}. + */ + public VideoHistogramSegmenter(ExtractionContextProvider context) { + this(context, new HashMap<>(0)); + } - private static Histogram getHistogram(VideoFrame f){ - return FuzzyColorHistogramCalculator.getSubdividedHistogramNormalized(f.getImage().getThumbnailImage(), 3); + /** + * Constructor required for instantiates through {@link ReflectionHelper}. + */ + public VideoHistogramSegmenter(ExtractionContextProvider context, Map parameters) { + if (parameters.containsKey("threshold")) { + this.threshold = Double.parseDouble(parameters.get("threshold")); + } else { + this.threshold = DEFAULT_THRESHOLD; } - - /** - * Method used to initialize the Segmenter - assigns the new Decoder instance and clears - * all the queues. - * - * @param object Media object that is about to be segmented. - */ - @Override - public synchronized void init(Decoder decoder, MediaObjectDescriptor object) { - if (!this.isrunning) { - this.decoder = decoder; - this.complete = false; - this.preShotList.clear(); - this.segments.clear(); - this.videoFrameList.clear(); - this.knownShotBoundaries.clear(); - this.knownShotBoundaries.addAll(this.segmentReader.lookUpSegmentsOfObject(object.getObjectId())); - this.knownShotBoundaries.sort(Comparator.comparingInt(MediaSegmentDescriptor::getSequenceNumber)); - } + if (parameters.containsKey("maxShotLength")) { + this.maxShotLength = Integer.parseInt(parameters.get("maxShotLength")); + } else { + this.maxShotLength = DEFAULT_MAX_SHOT_LENGTH; + } + this.segmentReader = new MediaSegmentReader(context.persistencyReader().get()); + } + + private static Histogram getHistogram(VideoFrame f) { + return FuzzyColorHistogramCalculator.getSubdividedHistogramNormalized(f.getImage().getThumbnailImage(), 3); + } + + /** + * Method used to initialize the Segmenter - assigns the new Decoder instance and clears all the queues. + * + * @param object Media object that is about to be segmented. + */ + @Override + public synchronized void init(Decoder decoder, MediaObjectDescriptor object) { + if (!this.isrunning) { + this.decoder = decoder; + this.complete = false; + this.preShotList.clear(); + this.segments.clear(); + this.videoFrameList.clear(); + this.knownShotBoundaries.clear(); + this.knownShotBoundaries.addAll(this.segmentReader.lookUpSegmentsOfObject(object.getObjectId())); + this.knownShotBoundaries.sort(Comparator.comparingInt(MediaSegmentDescriptor::getSequenceNumber)); + } + } + + @Override + public SegmentContainer getNext() throws InterruptedException { + SegmentContainer nextContainer = this.segments.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); + if (nextContainer == null) { + synchronized (this) { + this.complete = !this.isrunning && this.decoder.complete(); + } + } + return nextContainer; + } + + /** + * Indicates whether the Segmenter is complete i.e. no new segments are to be expected. + */ + @Override + public synchronized boolean complete() { + return this.complete; + } + + /** + * + */ + @Override + public synchronized void close() { + if (!this.isrunning) { + if (this.decoder != null) { + this.decoder.close(); + } + } + } + + /** + * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's + * run method to be called in that separately executing + * thread. + *

    + * The general contract of the method run is that it may take any action whatsoever. + * + * @see Thread#run() + */ + @Override + public void run() { + /* Begin: Set running to false. */ + synchronized (this) { + this.isrunning = true; } - @Override - public SegmentContainer getNext() throws InterruptedException { - SegmentContainer nextContainer = this.segments.poll(SEGMENT_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); - if (nextContainer == null) { - synchronized (this) { - this.complete = !this.isrunning && this.decoder.complete(); - } + try { + while (!this.decoder.complete()) { + if (this.videoFrameList.isEmpty()) { + queueFrames(); } - return nextContainer; - } - /** - * Indicates whether the Segmenter is complete i.e. no new segments - * are to be expected. - */ - @Override - public synchronized boolean complete() { - return this.complete; - } + VideoSegment _return = null; - /** - * - */ - @Override - public synchronized void close() { - if (!this.isrunning) { - if(this.decoder != null){ - this.decoder.close(); - } + if (!preShotList.isEmpty()) { + _return = new VideoSegment(); + while (!preShotList.isEmpty()) { + _return.addVideoFrame(preShotList.removeFirst().first); + } } - } - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

    - * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - /* Begin: Set running to false. */ - synchronized(this) { - this.isrunning = true; + if (this.videoFrameList.isEmpty()) { + this.segments.put(_return); + continue; //no more shots to segment } - try { - while (!this.decoder.complete()) { - if (this.videoFrameList.isEmpty()) { - queueFrames(); - } + if (_return == null) { + _return = new VideoSegment(); + } - VideoSegment _return = null; + VideoFrame videoFrame = this.videoFrameList.poll(); - if (!preShotList.isEmpty()) { - _return = new VideoSegment(); - while (!preShotList.isEmpty()) { - _return.addVideoFrame(preShotList.removeFirst().first); - } - } + MediaSegmentDescriptor bounds = this.knownShotBoundaries.size() > 0 ? this.knownShotBoundaries.remove(0) : null; - if (this.videoFrameList.isEmpty()) { - this.segments.put(_return); - continue; //no more shots to segment - } + if (bounds != null && videoFrame.getId() >= bounds.getStart() && videoFrame.getId() <= bounds.getEnd()) { - if (_return == null) { - _return = new VideoSegment(); - } + _return.addVideoFrame(videoFrame); + queueFrames(bounds.getEnd() - bounds.getStart()); + do { + videoFrame = this.videoFrameList.poll(); + if (videoFrame != null) { + _return.addVideoFrame(videoFrame); + } else { + break; + } + } while (videoFrame.getId() < bounds.getEnd()); + this.segments.put(_return); + continue; - VideoFrame videoFrame = this.videoFrameList.poll(); - - MediaSegmentDescriptor bounds = this.knownShotBoundaries.size() > 0 ? this.knownShotBoundaries.remove(0) : null; - - if (bounds != null && videoFrame.getId() >= bounds.getStart() && videoFrame.getId() <= bounds.getEnd()) { - - _return.addVideoFrame(videoFrame); - queueFrames(bounds.getEnd() - bounds.getStart()); - do { - videoFrame = this.videoFrameList.poll(); - if (videoFrame != null) { - _return.addVideoFrame(videoFrame); - } else { - break; - } - } while (videoFrame.getId() < bounds.getEnd()); - - this.segments.put(_return); - continue; - - } else { - Histogram hPrev, h = getHistogram(videoFrame); - _return.addVideoFrame(videoFrame); - while (true) { - if ((videoFrame = this.videoFrameList.poll()) == null) { - queueFrames(); - if ((videoFrame = this.videoFrameList.poll()) == null) { - this.segments.put(_return); - _return = null; - break; - } - } - hPrev = h; - h = getHistogram(videoFrame); - double distance = hPrev.getDistance(h); - - preShotList.offer(new Pair<>(videoFrame, distance)); - - if (preShotList.size() > PRESHOT_QUEUE_LENGTH) { - double max = 0; - int index = -1, i = 0; - for (Pair pair : preShotList) { - if (pair.second > max) { - index = i; - max = pair.second; - } - i++; - } - if (max <= this.threshold && _return.getNumberOfFrames() < this.maxShotLength) { //no cut - for (Pair pair : preShotList) { - _return.addVideoFrame(pair.first); - } - preShotList.clear(); - } else { - for (i = 0; i < index; ++i) { - _return.addVideoFrame(preShotList.removeFirst().first); - } - break; - } - } - } - if(_return != null){ - this.segments.put(_return); - } + } else { + Histogram hPrev, h = getHistogram(videoFrame); + _return.addVideoFrame(videoFrame); + while (true) { + if ((videoFrame = this.videoFrameList.poll()) == null) { + queueFrames(); + if ((videoFrame = this.videoFrameList.poll()) == null) { + this.segments.put(_return); + _return = null; + break; + } + } + hPrev = h; + h = getHistogram(videoFrame); + double distance = hPrev.getDistance(h); + + preShotList.offer(new Pair<>(videoFrame, distance)); + + if (preShotList.size() > PRESHOT_QUEUE_LENGTH) { + double max = 0; + int index = -1, i = 0; + for (Pair pair : preShotList) { + if (pair.second > max) { + index = i; + max = pair.second; + } + i++; + } + if (max <= this.threshold && _return.getNumberOfFrames() < this.maxShotLength) { //no cut + for (Pair pair : preShotList) { + _return.addVideoFrame(pair.first); + } + preShotList.clear(); + } else { + for (i = 0; i < index; ++i) { + _return.addVideoFrame(preShotList.removeFirst().first); } + break; + } } - } catch (InterruptedException e) { - LOGGER.log(Level.ERROR, "The thread that runs the VideoHistogramSegmenter was interrupted: {}", e); - } - - /* End: Reset running to false. */ - synchronized(this) { - this.isrunning = false; + } + if (_return != null) { + this.segments.put(_return); + } } + } + } catch (InterruptedException e) { + LOGGER.log(Level.ERROR, "The thread that runs the VideoHistogramSegmenter was interrupted: {}", e); } - private boolean queueFrames(){ - return queueFrames(20); + /* End: Reset running to false. */ + synchronized (this) { + this.isrunning = false; } - - private boolean queueFrames(int number) { - for(int i = 0; i < number; ++i){ - VideoFrame f = this.decoder.getNext(); - if (f == null) { //no more frames - return false; - } else { - synchronized (this) { - this.videoFrameList.offer(f); - } - } + } + + private boolean queueFrames() { + return queueFrames(20); + } + + private boolean queueFrames(int number) { + for (int i = 0; i < number; ++i) { + VideoFrame f = this.decoder.getNext(); + if (f == null) { //no more frames + return false; + } else { + synchronized (this) { + this.videoFrameList.offer(f); } - return true; + } } + return true; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java index ea1854843..cda815204 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AudioFingerprint.java @@ -3,6 +3,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; import gnu.trove.list.array.TIntArrayList; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -18,200 +21,196 @@ import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - public class AudioFingerprint extends StagedFeatureModule { - /** Frequency-ranges that should be used to calculate the fingerprint. */ - private static final float[] RANGES = {30.0f, 40.0f, 80.0f, 120.0f, 180.0f, 300.0f, 480.0f}; - /** Length of an individual fingerprint (size of FV). */ - private static final int FINGERPRINT = 20 * (RANGES.length-1); - - /** Size of the window during STFT in seconds (as proposed in [2]). */ - private static final float WINDOW_SIZE = 0.2f; - - /** - * Default constructor; - */ - public AudioFingerprint() { - super("features_audiofingerprint", 4000.0f, FINGERPRINT); + /** + * Frequency-ranges that should be used to calculate the fingerprint. + */ + private static final float[] RANGES = {30.0f, 40.0f, 80.0f, 120.0f, 180.0f, 300.0f, 480.0f}; + + /** + * Length of an individual fingerprint (size of FV). + */ + private static final int FINGERPRINT = 20 * (RANGES.length - 1); + + /** + * Size of the window during STFT in seconds (as proposed in [2]). + */ + private static final float WINDOW_SIZE = 0.2f; + + /** + * Default constructor; + */ + public AudioFingerprint() { + super("features_audiofingerprint", 4000.0f, FINGERPRINT); + } + + @Override + public void processSegment(SegmentContainer segment) { + TIntArrayList filteredSpectrum = this.filterSpectrum(segment); + int vectors = filteredSpectrum.size() / FINGERPRINT; + List tuples = new ArrayList<>(); + for (int i = 0; i < vectors; i++) { + float[] feature = new float[FINGERPRINT]; + for (int j = 0; j < FINGERPRINT; j++) { + feature[j] = filteredSpectrum.get(i * FINGERPRINT + j); + } + tuples.add(this.phandler.generateTuple(segment.getId(), feature)); } - - @Override - public void processSegment(SegmentContainer segment) { - TIntArrayList filteredSpectrum = this.filterSpectrum(segment); - int vectors = filteredSpectrum.size()/FINGERPRINT; - List tuples = new ArrayList<>(); - for (int i = 0; i < vectors; i++) { - float[] feature = new float[FINGERPRINT]; - for (int j=0; j < FINGERPRINT; j++) { - feature[j] = filteredSpectrum.get(i * FINGERPRINT + j); - } - tuples.add(this.phandler.generateTuple(segment.getId(), feature)); + this.phandler.persist(tuples); + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + * + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + /* Prepare empty list of features. */ + List features = new ArrayList<>(); + + /* Extract filtered spectrum and create query-vectors. */ + final TIntArrayList filteredSpectrum = this.filterSpectrum(sc); + final int shift = RANGES.length - 1; + final int lookups = Math.max(1, Math.min(30, filteredSpectrum.size() - FINGERPRINT)); + + for (int i = 1; i <= lookups; i++) { + float[] feature = new float[FINGERPRINT]; + for (int j = 0; j < FINGERPRINT; j++) { + if (i * shift + j < filteredSpectrum.size()) { + feature[j] = filteredSpectrum.get(i * shift + j); } - this.phandler.persist(tuples); + } + features.add(feature); } - - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - * - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Prepare empty list of features. */ - List features = new ArrayList<>(); - - /* Extract filtered spectrum and create query-vectors. */ - final TIntArrayList filteredSpectrum = this.filterSpectrum(sc); - final int shift = RANGES.length-1; - final int lookups = Math.max(1,Math.min(30, filteredSpectrum.size() - FINGERPRINT)); - - for (int i=1;i<=lookups;i++) { - float[] feature = new float[FINGERPRINT]; - for (int j=0; j< FINGERPRINT; j++) { - if (i * shift + j < filteredSpectrum.size()) { - feature[j] = filteredSpectrum.get(i * shift + j); - } - } - features.add(feature); - } - return features; + return features; + } + + /** + * @param features A list of feature-vectors (usually generated in the first stage). For each feature, a lookup is executed. May be empty! + * @param configs A ReadableQueryConfig object that contains query-related configuration parameters. + */ + @Override + protected List lookup(List features, List configs) { + List partialResults; + if (features.size() == 1) { + partialResults = this.selector.getNearestNeighboursGeneric(configs.get(0).getResultsPerModule(), features.get(0), FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs.get(0)); + } else { + partialResults = this.selector.getBatchedNearestNeighbours(configs.get(0).getResultsPerModule(), features, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs); } - - /** - * - * @param features A list of feature-vectors (usually generated in the first stage). For each feature, a lookup is executed. May be empty! - * @param configs A ReadableQueryConfig object that contains query-related configuration parameters. - */ - @Override - protected List lookup(List features, List configs) { - List partialResults; - if (features.size() == 1) { - partialResults = this.selector.getNearestNeighboursGeneric(configs.get(0).getResultsPerModule(), features.get(0), FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs.get(0)); + return partialResults; + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Prepare empty list of results. */ + final ArrayList results = new ArrayList<>(); + final HashMap map = new HashMap<>(); + + /* Merge into map for final results; select the minimum distance. */ + for (DistanceElement result : partialResults) { + map.merge(result.getId(), result, (d1, d2) -> { + if (d1.getDistance() > d2.getDistance()) { + return d2; } else { - partialResults = this.selector.getBatchedNearestNeighbours(configs.get(0).getResultsPerModule(), features, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs); + return d1; } - return partialResults; + }); } - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Prepare empty list of results. */ - final ArrayList results = new ArrayList<>(); - final HashMap map = new HashMap<>(); - - /* Merge into map for final results; select the minimum distance. */ - for (DistanceElement result : partialResults) { - map.merge(result.getId(), result, (d1, d2)-> { - if (d1.getDistance() > d2.getDistance()) { - return d2; - } else { - return d1; - } - }); - } - - /* Return immediately if no partial results are available. */ - if (map.isEmpty()) { - return results; - } - - /* Prepare final results. */ - final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); - map.forEach((key, value) -> results.add(value.toScore(fkt))); - return ScoreElement.filterMaximumScores(results.stream()); + /* Return immediately if no partial results are available. */ + if (map.isEmpty()) { + return results; } - /** - * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the - * same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. - * - * @param qc Original query config - * @param features List of features for which a QueryConfig is required. - * @return New query config (may be identical to the original one). - */ - @Override - protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { - ArrayList configs = new ArrayList<>(features.size()); - for (float[] feature : features) { - float[] weight = new float[feature.length]; - for (int i=0;i 0) { - weight[i] = 1.0f; - } else if (feature[i] == 0) { - weight[i] = 0.1f; - } - } - configs.add(new QueryConfig(qc).setDistanceWeights(weight)); + /* Prepare final results. */ + final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); + map.forEach((key, value) -> results.add(value.toScore(fkt))); + return ScoreElement.filterMaximumScores(results.stream()); + } + + /** + * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. + * + * @param qc Original query config + * @param features List of features for which a QueryConfig is required. + * @return New query config (may be identical to the original one). + */ + @Override + protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { + ArrayList configs = new ArrayList<>(features.size()); + for (float[] feature : features) { + float[] weight = new float[feature.length]; + for (int i = 0; i < feature.length; i++) { + if (feature[i] > 0) { + weight[i] = 1.0f; + } else if (feature[i] == 0) { + weight[i] = 0.1f; } - return configs; + } + configs.add(new QueryConfig(qc).setDistanceWeights(weight)); } - - private TIntArrayList filterSpectrum(SegmentContainer segment) { - /* Prepare empty list of candidates for filtered spectrum. */ - TIntArrayList candidates = new TIntArrayList(); - - /* Perform STFT and extract the Spectra. If this fails, return empty list. */ - Pair properties = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); - STFT stft = segment.getSTFT(properties.first, (properties.first-2*properties.second)/2, properties.second, new HanningWindow()); - if (stft == null) { - return candidates; - } - List spectra = stft.getPowerSpectrum(); - - /* Foreach spectrum; find peak-values in the defined ranges. */ - for (Spectrum spectrum : spectra) { - int spectrumidx = 0; - for (int j = 0; j < RANGES.length - 1; j++) { - Pair peak = null; - for (int k = spectrumidx; k < spectrum.size(); k++) { - Pair bin = spectrum.get(k); - if (bin.first >= RANGES[j] && bin.first <= RANGES[j + 1]) { - if (peak == null || bin.second > peak.second) { - peak = bin; - } - } else if (bin.first > RANGES[j + 1]) { - spectrumidx = k; - break; - } - } - candidates.add(Math.round(peak.first - (peak.first.intValue() % 2))); + return configs; + } + + private TIntArrayList filterSpectrum(SegmentContainer segment) { + /* Prepare empty list of candidates for filtered spectrum. */ + TIntArrayList candidates = new TIntArrayList(); + + /* Perform STFT and extract the Spectra. If this fails, return empty list. */ + Pair properties = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); + STFT stft = segment.getSTFT(properties.first, (properties.first - 2 * properties.second) / 2, properties.second, new HanningWindow()); + if (stft == null) { + return candidates; + } + List spectra = stft.getPowerSpectrum(); + + /* Foreach spectrum; find peak-values in the defined ranges. */ + for (Spectrum spectrum : spectra) { + int spectrumidx = 0; + for (int j = 0; j < RANGES.length - 1; j++) { + Pair peak = null; + for (int k = spectrumidx; k < spectrum.size(); k++) { + Pair bin = spectrum.get(k); + if (bin.first >= RANGES[j] && bin.first <= RANGES[j + 1]) { + if (peak == null || bin.second > peak.second) { + peak = bin; } + } else if (bin.first > RANGES[j + 1]) { + spectrumidx = k; + break; + } } - return candidates; - } - - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. - */ - @Override - protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.manhattan) - .addHint(ReadableQueryConfig.Hints.inexact); + candidates.add(Math.round(peak.first - (peak.first.intValue() % 2))); + } } + return candidates; + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.manhattan) + .addHint(ReadableQueryConfig.Hints.inexact); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColor.java index a7667fec3..3ecaa336d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColor.java @@ -1,19 +1,18 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.ReadableLabContainer; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ColorUtils; -import java.util.List; - public class AverageColor extends AbstractFeatureModule { public AverageColor() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44.java index 81dc79ca5..638c401ac 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44.java @@ -1,20 +1,19 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; -import java.util.List; - public class AverageColorARP44 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44Normalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44Normalized.java index 4f97a6c92..11d7af4c5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44Normalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorARP44Normalized.java @@ -1,21 +1,20 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class AverageColorARP44Normalized extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLD.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLD.java index 91d80a5f3..5da12ccb0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLD.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLD.java @@ -1,18 +1,17 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; -import java.util.List; - public class AverageColorCLD extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLDNormalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLDNormalized.java index 12b1e24d9..cb58c6af6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLDNormalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorCLDNormalized.java @@ -1,19 +1,18 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class AverageColorCLDNormalized extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java index 3f3188ccf..8f14346a9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; @@ -7,7 +10,10 @@ import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.*; +import org.vitrivr.cineast.core.data.FloatVector; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -15,10 +21,6 @@ import org.vitrivr.cineast.core.util.ColorUtils; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class AverageColorGrid8 extends AbstractFeatureModule { public AverageColorGrid8() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Normalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Normalized.java index 195296d7a..a685edce5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Normalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Normalized.java @@ -1,19 +1,18 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class AverageColorGrid8Normalized extends AverageColorGrid8 { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced11.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced11.java index b131d57c1..94ab5148d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced11.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced11.java @@ -1,19 +1,18 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.List; - public class AverageColorGrid8Reduced11 extends AverageColorGrid8 { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced15.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced15.java index 5d0ba3223..dae0c5c69 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced15.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8Reduced15.java @@ -1,19 +1,18 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.List; - public class AverageColorGrid8Reduced15 extends AverageColorGrid8 { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java index a942417b9..d614be5fb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java @@ -2,6 +2,11 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; @@ -12,11 +17,11 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.score.SegmentScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -29,12 +34,6 @@ import org.vitrivr.cineast.core.util.ColorUtils; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - public class AverageColorRaster extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); @@ -51,75 +50,75 @@ public void init(PersistencyWriterSupplier supply, int batchSize) { protected static int get(Color c) { switch (c) { - case Black: - return 0; - case Blue: - return 1; - case Brown: - return 2; - case Cyan: - return 3; - case Green: - return 4; - case Grey: - return 5; - case Magenta: - return 6; - case Navy: - return 7; - case Orange: - return 8; - case Pink: - return 9; - case Red: - return 10; - case Teal: - return 11; - case Violet: - return 12; - case White: - return 13; - case Yellow: - return 14; - default: - return -1; + case Black: + return 0; + case Blue: + return 1; + case Brown: + return 2; + case Cyan: + return 3; + case Green: + return 4; + case Grey: + return 5; + case Magenta: + return 6; + case Navy: + return 7; + case Orange: + return 8; + case Pink: + return 9; + case Red: + return 10; + case Teal: + return 11; + case Violet: + return 12; + case White: + return 13; + case Yellow: + return 14; + default: + return -1; } } protected static Color get(int i) { switch (i) { - case 0: - return Color.Black; - case 1: - return Color.Blue; - case 2: - return Color.Brown; - case 3: - return Color.Cyan; - case 4: - return Color.Green; - case 5: - return Color.Grey; - case 6: - return Color.Magenta; - case 7: - return Color.Navy; - case 8: - return Color.Orange; - case 9: - return Color.Pink; - case 10: - return Color.Red; - case 11: - return Color.Teal; - case 12: - return Color.Violet; - case 13: - return Color.White; - case 14: - return Color.Yellow; - default: - return Color.Black; + case 0: + return Color.Black; + case 1: + return Color.Blue; + case 2: + return Color.Brown; + case 3: + return Color.Cyan; + case 4: + return Color.Green; + case 5: + return Color.Grey; + case 6: + return Color.Magenta; + case 7: + return Color.Navy; + case 8: + return Color.Orange; + case 9: + return Color.Pink; + case 10: + return Color.Red; + case 11: + return Color.Teal; + case 12: + return Color.Violet; + case 13: + return Color.White; + case 14: + return Color.Yellow; + default: + return Color.Black; } } @@ -204,56 +203,56 @@ protected static double score(float f1, float f2) { return 1d; } switch (c1) { - case Black: - if (c2 == Color.Grey) { - return 0.25; - } - break; - case Blue: - if (c2 == Color.Navy || c2 == Color.Violet) { - return 0.5; - } - if (c2 == Color.Cyan) { - return 0.25; - } - break; - case Brown: - if (c2 == Color.Grey) { - return 0.5; - } - break; - case Cyan: - if (c2 == Color.White) { - return 0.25; - } - break; - case Green: - if (c2 == Color.Teal) { - return 0.5; - } - break; - case Grey: - if (c2 == Color.White || c2 == Color.Black) { - return 0.125; - } - break; - case Magenta: - if (c2 == Color.Violet || c2 == Color.Pink) { - return 0.5; - } - break; - case Orange: - if (c2 == Color.Red || c2 == Color.Yellow) { - return 0.5; - } - break; - case Pink: - if (c2 == Color.Red) { - return 0.5; - } - break; - default: - return 0; + case Black: + if (c2 == Color.Grey) { + return 0.25; + } + break; + case Blue: + if (c2 == Color.Navy || c2 == Color.Violet) { + return 0.5; + } + if (c2 == Color.Cyan) { + return 0.25; + } + break; + case Brown: + if (c2 == Color.Grey) { + return 0.5; + } + break; + case Cyan: + if (c2 == Color.White) { + return 0.25; + } + break; + case Green: + if (c2 == Color.Teal) { + return 0.5; + } + break; + case Grey: + if (c2 == Color.White || c2 == Color.Black) { + return 0.125; + } + break; + case Magenta: + if (c2 == Color.Violet || c2 == Color.Pink) { + return 0.5; + } + break; + case Orange: + if (c2 == Color.Red || c2 == Color.Yellow) { + return 0.5; + } + break; + case Pink: + if (c2 == Color.Red) { + return 0.5; + } + break; + default: + return 0; } return 0; } @@ -288,7 +287,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc @Override public List getSimilar(String segmentId, ReadableQueryConfig qc) { - List> rows = this.selector.getRows(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId)); + List> rows = this.selector.getRows(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId)); if (rows.isEmpty()) { return new ArrayList<>(1); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java index c27d277ff..2bc6b43d0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java @@ -2,6 +2,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.function.Supplier; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; @@ -10,30 +11,28 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.function.Supplier; - public class AverageColorRasterReduced11 extends AverageColorRaster { - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - /* TODO: Respect batchSize. */ - this.phandler = supply.get(); - this.phandler.open("features_AverageColorRasterReduced11"); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); - } + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + /* TODO: Respect batchSize. */ + this.phandler = supply.get(); + this.phandler.open("features_AverageColorRasterReduced11"); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); + } + + @Override + MultiImage getMultiImage(SegmentContainer shot) { + return ColorReductionUtil.quantize11(shot.getAvgImg()); + } - @Override - MultiImage getMultiImage(SegmentContainer shot){ - return ColorReductionUtil.quantize11(shot.getAvgImg()); - } - - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity("features_AverageColorRasterReduced11", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); - } + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity("features_AverageColorRasterReduced11", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); + } - @Override - public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity("features_AverageColorRasterReduced11"); - } + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity("features_AverageColorRasterReduced11"); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java index 732adda0c..fdc90329b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java @@ -2,6 +2,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.function.Supplier; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; @@ -10,30 +11,28 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.function.Supplier; - public class AverageColorRasterReduced15 extends AverageColorRaster { - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - /* TODO: Respect batchSize. */ - this.phandler = supply.get(); - this.phandler.open("features_AverageColorRasterReduced15"); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); - } + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + /* TODO: Respect batchSize. */ + this.phandler = supply.get(); + this.phandler.open("features_AverageColorRasterReduced15"); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); + } + + @Override + MultiImage getMultiImage(SegmentContainer shot) { + return ColorReductionUtil.quantize15(shot.getAvgImg()); + } - @Override - MultiImage getMultiImage(SegmentContainer shot){ - return ColorReductionUtil.quantize15(shot.getAvgImg()); - } - - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity("features_AverageColorRasterReduced15", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); - } + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity("features_AverageColorRasterReduced15", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); + } - @Override - public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity("features_AverageColorRasterReduced15"); - } + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity("features_AverageColorRasterReduced15"); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHist.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHist.java index 5f51453b0..662cef886 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHist.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHist.java @@ -1,20 +1,19 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.extraction.segmenter.FuzzyColorHistogram; import org.vitrivr.cineast.core.extraction.segmenter.FuzzyColorHistogramCalculator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class AverageFuzzyHist extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHistNormalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHistNormalized.java index 9a0dad79c..5af051115 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHistNormalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageFuzzyHistNormalized.java @@ -1,12 +1,13 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.extraction.segmenter.FuzzyColorHistogram; @@ -14,8 +15,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class AverageFuzzyHistNormalized extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP.java index 424a51825..91a6d7c2e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -16,136 +19,136 @@ import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - public abstract class AverageHPCP extends StagedFeatureModule { - /** Size of the window during STFT in samples. */ - private final static float WINDOW_SIZE = 0.2f; - - /** Minimum resolution to consider in HPCP calculation. */ - private final float min_frequency; - - /** Maximum resolution to consider in HPCP calculation. */ - private final float max_frequency; - - /** HPCP resolution (12, 24, 36 bins). */ - private final HPCP.Resolution resolution; - - /* The number of HPCP frames that should be averaged into a single vector. */ - private final int average; - - /** - * Default constructor. - * - * @param name Name of the entity (for persistence writer). - * @param min_frequency Minimum frequency to consider for HPCP generation. - * @param max_frequency Maximum frequency to consider for HPCP generation. - * @param resolution Resolution of HPCP (i.e. number of HPCP bins). - * @param average Number of frames to average. - */ - protected AverageHPCP(String name, float min_frequency, float max_frequency, HPCP.Resolution resolution, int average) { - super(name, 2.0f, 2*resolution.bins); - - /* Assign variables. */ - this.min_frequency = min_frequency; - this.max_frequency = max_frequency; - this.resolution = resolution; - this.average = average; - } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module. - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return List of feature vectors for lookup. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - return this.getFeatures(sc); + /** + * Size of the window during STFT in samples. + */ + private final static float WINDOW_SIZE = 0.2f; + + /** + * Minimum resolution to consider in HPCP calculation. + */ + private final float min_frequency; + + /** + * Maximum resolution to consider in HPCP calculation. + */ + private final float max_frequency; + + /** + * HPCP resolution (12, 24, 36 bins). + */ + private final HPCP.Resolution resolution; + + /* The number of HPCP frames that should be averaged into a single vector. */ + private final int average; + + /** + * Default constructor. + * + * @param name Name of the entity (for persistence writer). + * @param min_frequency Minimum frequency to consider for HPCP generation. + * @param max_frequency Maximum frequency to consider for HPCP generation. + * @param resolution Resolution of HPCP (i.e. number of HPCP bins). + * @param average Number of frames to average. + */ + protected AverageHPCP(String name, float min_frequency, float max_frequency, HPCP.Resolution resolution, int average) { + super(name, 2.0f, 2 * resolution.bins); + + /* Assign variables. */ + this.min_frequency = min_frequency; + this.max_frequency = max_frequency; + this.resolution = resolution; + this.average = average; + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module. + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return List of feature vectors for lookup. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + return this.getFeatures(sc); + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Prepare helper data-structures. */ + final HashMap map = new HashMap<>(); + + /* Set QueryConfig and extract correspondence function. */ + for (DistanceElement hit : partialResults) { + map.merge(hit.getId(), hit, (o, n) -> new SegmentDistanceElement(hit.getId(), (o.getDistance() + n.getDistance()) / 2)); } - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Prepare helper data-structures. */ - final HashMap map = new HashMap<>(); - - /* Set QueryConfig and extract correspondence function. */ - for (DistanceElement hit : partialResults) { - map.merge(hit.getId(), hit, (o,n) -> new SegmentDistanceElement(hit.getId(), (o.getDistance() + n.getDistance())/2)); - } - - /* Prepare final result-set. */ - final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(map.entrySet().stream().map(e -> e.getValue().toScore(fkt))); + /* Prepare final result-set. */ + final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(map.entrySet().stream().map(e -> e.getValue().toScore(fkt))); + } + + /** + * Processes a SegmentContainer during extraction; calculates the HPCP for the container and creates and persists a set shingle-vectors. + * + * @param segment SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer segment) { + List list = this.getFeatures(segment); + list.forEach(f -> this.persist(segment.getId(), new FloatVectorImpl(f))); + } + + /** + * Returns a list of feature vectors given a SegmentContainer. + * + * @param segment SegmentContainer for which to calculate the feature vectors. + * @return List of HPCP Shingle feature vectors. + */ + private List getFeatures(SegmentContainer segment) { + /* Create STFT. IF this fails, return empty list. */ + Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); + STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow()); + if (stft == null) { + return new ArrayList<>(); } - /** - * Processes a SegmentContainer during extraction; calculates the HPCP for the container and - * creates and persists a set shingle-vectors. - * - * @param segment SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer segment) { - List list = this.getFeatures(segment); - list.forEach(f -> this.persist(segment.getId(), new FloatVectorImpl(f))); - } - - /** - * Returns a list of feature vectors given a SegmentContainer. - * - * @param segment SegmentContainer for which to calculate the feature vectors. - * @return List of HPCP Shingle feature vectors. - */ - private List getFeatures(SegmentContainer segment) { - /* Create STFT. IF this fails, return empty list. */ - Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); - STFT stft = segment.getSTFT(parameters.first, (parameters.first-2*parameters.second)/2,parameters.second, new HanningWindow()); - if (stft == null) { - return new ArrayList<>(); - } - - HPCP hpcps = new HPCP(this.resolution, this.min_frequency, this.max_frequency); - hpcps.addContribution(stft); - - /* Determine number of vectors that will result from the data. */ - int vectors = hpcps.size()/this.average; - - List features = new ArrayList<>(vectors); - for (int i = 0; i < vectors; i++) { - float[] feature = new float[2*this.resolution.bins]; - SummaryStatistics[] statistics = new SummaryStatistics[this.resolution.bins]; - for (int j = 0; j features = new ArrayList<>(vectors); + for (int i = 0; i < vectors; i++) { + float[] feature = new float[2 * this.resolution.bins]; + SummaryStatistics[] statistics = new SummaryStatistics[this.resolution.bins]; + for (int j = 0; j < this.average; j++) { + for (int k = 0; k < this.resolution.bins; k++) { + if (statistics[k] == null) { + statistics[k] = new SummaryStatistics(); + } + statistics[k].addValue(hpcps.getHpcp(i * this.average + j)[k]); } - return features; + } + for (int k = 0; k < this.resolution.bins; k++) { + feature[2 * k] = (float) statistics[k].getMean(); + feature[2 * k + 1] = (float) statistics[k].getStandardDeviation(); + } + features.add(MathHelper.normalizeL2(feature)); } + return features; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP20F36B.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP20F36B.java index 5e03fcc4d..7768c3983 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP20F36B.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP20F36B.java @@ -4,14 +4,15 @@ public class AverageHPCP20F36B extends AverageHPCP { - private final static float MIN_FREQUENCY = 50.0f; - private final static float MAX_FREQUENCY = 5000.0f; + private final static float MIN_FREQUENCY = 50.0f; - /** - * Default constructor. - */ - public AverageHPCP20F36B() { - super("features_avghpcp20f36b", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.THIRDSEMITON, 20); - } + private final static float MAX_FREQUENCY = 5000.0f; + + /** + * Default constructor. + */ + public AverageHPCP20F36B() { + super("features_avghpcp20f36b", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.THIRDSEMITON, 20); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP30F36B.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP30F36B.java index 7e5600921..99badb520 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP30F36B.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageHPCP30F36B.java @@ -4,14 +4,15 @@ public class AverageHPCP30F36B extends AverageHPCP { - private final static float MIN_FREQUENCY = 50.0f; - private final static float MAX_FREQUENCY = 5000.0f; + private final static float MIN_FREQUENCY = 50.0f; - /** - * Default constructor. - */ - public AverageHPCP30F36B() { - super("features_avghpcp30f36b", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.THIRDSEMITON, 30); - } + private final static float MAX_FREQUENCY = 5000.0f; + + /** + * Default constructor. + */ + public AverageHPCP30F36B() { + super("features_avghpcp30f36b", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.THIRDSEMITON, 30); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java index 78b23c21b..a592ba42f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java @@ -1,6 +1,9 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -18,195 +21,185 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.BlackmanHarrisWindow; import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - /** - * An Extraction and Retrieval module that leverages pure HPCP based CENS shingles according to [1]. These shingles can be used - * for cover-song and version identification. - * - * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. - * In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 - * - * [2] Kurth, F., & Müler, M. (2008). Efficient index-based audio matching. - * IEEE Transactions on Audio, Speech and Language Processing, 16(2), 382–395. http://doi.org/10.1109/TASL.2007.911552 - * + * An Extraction and Retrieval module that leverages pure HPCP based CENS shingles according to [1]. These shingles can be used for cover-song and version identification. + *

    + * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 + *

    + * [2] Kurth, F., & Müler, M. (2008). Efficient index-based audio matching. IEEE Transactions on Audio, Speech and Language Processing, 16(2), 382–395. http://doi.org/10.1109/TASL.2007.911552 */ public abstract class CENS extends StagedFeatureModule { - /** Size of the window during STFT in seconds (as proposed in [2]). */ - private final static float WINDOW_SIZE = 0.1f; - - /** - * Size of a shingle in number of HPCP features. - * - * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components. - */ - private final static int SHINGLE_SIZE = 10; - - /** - * Array of CENS(w,d) settings used at query time. One CENS feature is calculated and looked-up for every entry - * in this array. The inner array contains the following parameters: - * - * 0 - Window size (w) - * 1 - Downsampling ratio (d) - */ - private final static int[][] QUERY_SETTINGS = {{11,2}, {21,5}, {41,10}, {81,20}}; - - /** Maximum resolution to consider in HPCP calculation. */ - private final float minFrequency; - - /** Minimum resolution to consider in HPCP calculation. */ - private final float maxFrequency; - - /** - * Default constructor. - * - * @param tableName Name of the entity (for persistence writer). - * - * @param minFrequency Minimum frequency to consider during HPCP analysis. - * @param maxFrequency Maximum frequency to consider during HPCP analysis. - */ - public CENS(String tableName, float minFrequency, float maxFrequency) { - super(tableName, 2.0f, SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins); - - /* Apply fields. */ - this.minFrequency = minFrequency; - this.maxFrequency = maxFrequency; + /** + * Size of the window during STFT in seconds (as proposed in [2]). + */ + private final static float WINDOW_SIZE = 0.1f; + + /** + * Size of a shingle in number of HPCP features. + *

    + * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components. + */ + private final static int SHINGLE_SIZE = 10; + + /** + * Array of CENS(w,d) settings used at query time. One CENS feature is calculated and looked-up for every entry in this array. The inner array contains the following parameters: + *

    + * 0 - Window size (w) 1 - Downsampling ratio (d) + */ + private final static int[][] QUERY_SETTINGS = {{11, 2}, {21, 5}, {41, 10}, {81, 20}}; + + /** + * Maximum resolution to consider in HPCP calculation. + */ + private final float minFrequency; + + /** + * Minimum resolution to consider in HPCP calculation. + */ + private final float maxFrequency; + + /** + * Default constructor. + * + * @param tableName Name of the entity (for persistence writer). + * @param minFrequency Minimum frequency to consider during HPCP analysis. + * @param maxFrequency Maximum frequency to consider during HPCP analysis. + */ + public CENS(String tableName, float minFrequency, float maxFrequency) { + super(tableName, 2.0f, SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins); + + /* Apply fields. */ + this.minFrequency = minFrequency; + this.maxFrequency = maxFrequency; + } + + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + * + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module. + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return List of feature vectors for lookup. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + + /* Create STFT. If this fails, return empty list. */ + final Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); + final STFT stft = sc.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 3, parameters.second, new BlackmanHarrisWindow()); + if (stft == null) { + return new ArrayList<>(0); } + /* Prepare empty features. */ + final List features = new ArrayList<>(3 * QUERY_SETTINGS.length); - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - * - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module. - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return List of feature vectors for lookup. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - - /* Create STFT. If this fails, return empty list. */ - final Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); - final STFT stft = sc.getSTFT(parameters.first, (parameters.first-2*parameters.second)/3 ,parameters.second, new BlackmanHarrisWindow()); - if (stft == null) { - return new ArrayList<>(0); - } - - /* Prepare empty features. */ - final List features = new ArrayList<>(3 * QUERY_SETTINGS.length); - - /* Prepare HPCPs... */ - final HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency); - hpcps.addContribution(stft); - - /* - * ... and derive CENS features; only three features per QUERY_SETTING are kept because the shifts - * resulting from the Shingeling are realised as individual vectors during ingest! - */ - for (int[] QUERY_SETTING : QUERY_SETTINGS) { - List cens = this.getFeatures(hpcps, QUERY_SETTING[0], QUERY_SETTING[1]); - for (int i=0; i postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Prepare map to build a unique set of results. */ - final HashMap map = new HashMap<>(); - for (DistanceElement hit : partialResults) { - map.merge(hit.getId(), hit, (o, n) -> o.getDistance() > n.getDistance() ? n : o); + for (int[] QUERY_SETTING : QUERY_SETTINGS) { + List cens = this.getFeatures(hpcps, QUERY_SETTING[0], QUERY_SETTING[1]); + for (int i = 0; i < cens.size(); i++) { + if (i % SHINGLE_SIZE == 0) { + features.add(cens.get(i)); } - - /* Prepare final list of results. */ - final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(map.entrySet().stream().map((e) -> e.getValue().toScore(correspondence))); + } } - /** - * Processes a SegmentContainer for later persisting it in the storage layer. - * - * @param sc The SegmentContainer that should be processed. - */ - @Override - public void processSegment(SegmentContainer sc) { - /* Create STFT. If this fails, return empty list. */ - Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); - STFT stft = sc.getSTFT(parameters.first, (parameters.first-2*parameters.second)/3, parameters.second, new HanningWindow()); - if (stft == null) { - return; - } - - /* Prepare HPCPs... */ - HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency); - hpcps.addContribution(stft); - - List features = this.getFeatures(hpcps, 41, 10); - features.forEach(f -> this.persist(sc.getId(), new FloatVectorImpl(f))); + return features; + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Prepare map to build a unique set of results. */ + final HashMap map = new HashMap<>(); + for (DistanceElement hit : partialResults) { + map.merge(hit.getId(), hit, (o, n) -> o.getDistance() > n.getDistance() ? n : o); } - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. - */ - @Override - protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.euclidean) - .addHint(ReadableQueryConfig.Hints.inexact); + /* Prepare final list of results. */ + final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(map.entrySet().stream().map((e) -> e.getValue().toScore(correspondence))); + } + + /** + * Processes a SegmentContainer for later persisting it in the storage layer. + * + * @param sc The SegmentContainer that should be processed. + */ + @Override + public void processSegment(SegmentContainer sc) { + /* Create STFT. If this fails, return empty list. */ + Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); + STFT stft = sc.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 3, parameters.second, new HanningWindow()); + if (stft == null) { + return; } - /** - * Returns a list of CENS shingle feature vectors given a SegmentContainer. - * - * @param hpcps HPCP from which to derive the feature vectors. - * @param w Window size w for CENS(w,d) - * @param downsample Downsample ratio d for CENS(w,d) - * @return List of CENS Shingle feature vectors. - */ - private List getFeatures(HPCP hpcps, int w, int downsample) { - /* Prepare empty list of results. */ - List features = new ArrayList<>(); - - double[][] cens = org.vitrivr.cineast.core.util.audio.CENS.cens(hpcps, w, downsample); - int numberoffeatures = cens.length - SHINGLE_SIZE + 1; - - for (int i = 0; i < numberoffeatures; i++) { - float[] feature = new float[SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins]; - for (int j = 0; j < SHINGLE_SIZE; j++) { - for (int k = 0; k < HPCP.Resolution.FULLSEMITONE.bins; k++) { - feature[j*HPCP.Resolution.FULLSEMITONE.bins + k] = (float)cens[j + i][k]; - } - } - if (MathHelper.checkNotZero(feature) && MathHelper.checkNotNaN(feature)) { - features.add(MathHelper.normalizeL2(feature)); - } + /* Prepare HPCPs... */ + HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency); + hpcps.addContribution(stft); + + List features = this.getFeatures(hpcps, 41, 10); + features.forEach(f -> this.persist(sc.getId(), new FloatVectorImpl(f))); + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean) + .addHint(ReadableQueryConfig.Hints.inexact); + } + + /** + * Returns a list of CENS shingle feature vectors given a SegmentContainer. + * + * @param hpcps HPCP from which to derive the feature vectors. + * @param w Window size w for CENS(w,d) + * @param downsample Downsample ratio d for CENS(w,d) + * @return List of CENS Shingle feature vectors. + */ + private List getFeatures(HPCP hpcps, int w, int downsample) { + /* Prepare empty list of results. */ + List features = new ArrayList<>(); + + double[][] cens = org.vitrivr.cineast.core.util.audio.CENS.cens(hpcps, w, downsample); + int numberoffeatures = cens.length - SHINGLE_SIZE + 1; + + for (int i = 0; i < numberoffeatures; i++) { + float[] feature = new float[SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins]; + for (int j = 0; j < SHINGLE_SIZE; j++) { + for (int k = 0; k < HPCP.Resolution.FULLSEMITONE.bins; k++) { + feature[j * HPCP.Resolution.FULLSEMITONE.bins + k] = (float) cens[j + i][k]; } - - return features; + } + if (MathHelper.checkNotZero(feature) && MathHelper.checkNotNaN(feature)) { + features.add(MathHelper.normalizeL2(feature)); + } } + + return features; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12BasslineShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12BasslineShingle.java index d0d320d91..22c9eea07 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12BasslineShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12BasslineShingle.java @@ -1,26 +1,22 @@ package org.vitrivr.cineast.core.features; /** - * An Extraction and Retrieval module that leverages HPCP CENS shingles according to [1]. The particular module - * focuses on bassline frequencies following an idea found in [2]. - * - * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. - * In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 - * - * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. - * International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 - * + * An Extraction and Retrieval module that leverages HPCP CENS shingles according to [1]. The particular module focuses on bassline frequencies following an idea found in [2]. + *

    + * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 + *

    + * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 */ public class CENS12BasslineShingle extends CENS { - private final static float MIN_FREQUENCY = 10.0f; + private final static float MIN_FREQUENCY = 10.0f; - private final static float MAX_FREQUENCY = 262.0f; + private final static float MAX_FREQUENCY = 262.0f; - /** - * Default constructor; CENS Shingle using frequencies between 5.0f and 262.0f Hz. - */ - public CENS12BasslineShingle() { - super("features_cens12bassline", MIN_FREQUENCY, MAX_FREQUENCY); - } + /** + * Default constructor; CENS Shingle using frequencies between 5.0f and 262.0f Hz. + */ + public CENS12BasslineShingle() { + super("features_cens12bassline", MIN_FREQUENCY, MAX_FREQUENCY); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12MelodyShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12MelodyShingle.java index 3d9c66c53..3448fbde8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12MelodyShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12MelodyShingle.java @@ -1,26 +1,22 @@ package org.vitrivr.cineast.core.features; /** - * An Extraction and Retrieval module that leverages HPCP CENS shingles according to [1]. The particular module - * focuses on melodic frequencies following an idea found in [2]. - * - * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. - * In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 - * - * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. - * International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 - * + * An Extraction and Retrieval module that leverages HPCP CENS shingles according to [1]. The particular module focuses on melodic frequencies following an idea found in [2]. + *

    + * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 + *

    + * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 */ public class CENS12MelodyShingle extends CENS { - private final static float MIN_FREQUENCY = 262.0f; + private final static float MIN_FREQUENCY = 262.0f; - private final static float MAX_FREQUENCY = 5000.0f; + private final static float MAX_FREQUENCY = 5000.0f; - /** - * Default constructor; CENS Shingle using frequencies between 262.0f and 5000.0f Hz. - */ - public CENS12MelodyShingle() { - super("features_cens12melody", MIN_FREQUENCY, MAX_FREQUENCY); - } + /** + * Default constructor; CENS Shingle using frequencies between 262.0f and 5000.0f Hz. + */ + public CENS12MelodyShingle() { + super("features_cens12melody", MIN_FREQUENCY, MAX_FREQUENCY); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12Shingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12Shingle.java index c3a2ea12c..918dd6ed9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12Shingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS12Shingle.java @@ -2,22 +2,21 @@ /** * An Extraction and Retrieval module that leverages HPCP CENS shingles according to [1]. - * - * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. - * In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 - * + *

    + * [1] Grosche, P., & Muller, M. (2012). Toward characteristic audio shingles for efficient cross-version music retrieval. In 2012 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (pp. 473–476). IEEE. http://doi.org/10.1109/ICASSP.2012.6287919 */ public class CENS12Shingle extends CENS { - /* Minimum frequency to consider. */ - private final static float MIN_FREQUENCY = 50.0f; - /* Maximum frequency to consider. */ - private final static float MAX_FREQUENCY = 5000.0f; + /* Minimum frequency to consider. */ + private final static float MIN_FREQUENCY = 50.0f; - /** - * Default constructor; CENS Shingle using frequencies between 5.0f and 262.0f Hz. - */ - public CENS12Shingle() { - super("features_cens12", MIN_FREQUENCY, MAX_FREQUENCY); - } + /* Maximum frequency to consider. */ + private final static float MAX_FREQUENCY = 5000.0f; + + /** + * Default constructor; CENS Shingle using frequencies between 5.0f and 262.0f Hz. + */ + public CENS12Shingle() { + super("features_cens12", MIN_FREQUENCY, MAX_FREQUENCY); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLD.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLD.java index 2e8e69455..9741d4d94 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLD.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLD.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.ReadableFloatVector; @@ -9,8 +10,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; -import java.util.List; - public class CLD extends AbstractFeatureModule { public CLD() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDNormalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDNormalized.java index 0027abb9b..7044a131e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDNormalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDNormalized.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.ReadableFloatVector; @@ -10,8 +11,6 @@ import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class CLDNormalized extends AbstractFeatureModule { public CLDNormalized() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced11.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced11.java index 917fcd289..82d48d28d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced11.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced11.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.ReadableFloatVector; @@ -10,8 +11,6 @@ import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.List; - public class CLDReduced11 extends AbstractFeatureModule { public CLDReduced11() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced15.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced15.java index 7ee0c17f0..d3708a914 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced15.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLDReduced15.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.ReadableFloatVector; @@ -10,8 +11,6 @@ import org.vitrivr.cineast.core.util.ColorLayoutDescriptor; import org.vitrivr.cineast.core.util.ColorReductionUtil; -import java.util.List; - public class CLDReduced15 extends AbstractFeatureModule { public CLDReduced15() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ChromaGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ChromaGrid8.java index 996b352b3..12539fbfb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ChromaGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ChromaGrid8.java @@ -1,5 +1,9 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,7 +11,10 @@ import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.*; +import org.vitrivr.cineast.core.data.FloatVector; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; @@ -15,11 +22,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - public class ChromaGrid8 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasks.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasks.java index fab893737..5dfbbf8bb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasks.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasks.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; import org.tensorflow.types.TUint8; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -10,11 +16,13 @@ import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.*; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLab; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabAde20k; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabCityscapes; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel; +import org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabPascalVoc; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.*; - public class ConceptMasks extends AbstractFeatureModule { private static final int GRID_PARTITIONS = 16; @@ -88,7 +96,6 @@ public synchronized void processSegment(SegmentContainer shot) { @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - Optional optional = sc.getSemanticMap(); if (!optional.isPresent()) { return Collections.emptyList(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasksAde20k.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasksAde20k.java index c93778ae5..c5d83b4f3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasksAde20k.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ConceptMasksAde20k.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tensorflow.types.TUint8; @@ -18,8 +24,6 @@ import org.vitrivr.cineast.core.util.GridPartitioner; import org.vitrivr.cineast.core.util.LogHelper; -import java.util.*; - public class ConceptMasksAde20k extends AbstractFeatureModule { private static final int GRID_PARTITIONS = 32; @@ -71,7 +75,6 @@ public synchronized void processSegment(SegmentContainer shot) { ArrayList> ade20kPartitions = GridPartitioner .partition(ade20kLabels, tmp.length, tmp[0].length, GRID_PARTITIONS, GRID_PARTITIONS); - float[] vector = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS]; for (int i = 0; i < GRID_PARTITIONS * GRID_PARTITIONS; ++i) { @@ -87,7 +90,6 @@ public synchronized void processSegment(SegmentContainer shot) { @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - Optional optional = sc.getSemanticMap(); if (!optional.isPresent()) { return Collections.emptyList(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java index 88bb230be..955afee3c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java @@ -7,6 +7,7 @@ import boofcv.alg.filter.blur.BlurImageOps; import boofcv.struct.image.GrayF32; import com.googlecode.javaewah.datastructure.BitSet; +import java.util.List; import java.util.function.Supplier; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.stat.descriptive.rank.Median; @@ -17,9 +18,9 @@ import org.ejml.dense.row.CommonOps_DDRM; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.providers.primitive.BitSetTypeProvider; import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; @@ -29,13 +30,10 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - /** * Simple Image Fingerprinting Feature. Re-implemented based on the following two papers: - * + *

    * Christoph Zauner. Implementation and benchmarking of perceptual image hash func- tions. Master’s thesis, University of Applied Sciences Hagenberg, Austria, 2010. Baris Coskun and Bulent Sankur. Robust video hash extraction. In Signal Processing Conference, 2004 12th European, pages 2295–2298. IEEE, 2004. - * */ public class DCTImageHash extends AbstractFeatureModule { @@ -149,8 +147,8 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc @Override public void initalizePersistentLayer(Supplier supply) { supply.get().createEntity(this.tableName, - new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), - new AttributeDefinition(FEATURE_COLUMN_QUALIFIER, AttributeType.BITSET, 64) + new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), + new AttributeDefinition(FEATURE_COLUMN_QUALIFIER, AttributeType.BITSET, 64) ); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyCollectionBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyCollectionBooleanRetriever.java index 899fb615c..65d660cf1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyCollectionBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyCollectionBooleanRetriever.java @@ -1,11 +1,11 @@ package org.vitrivr.cineast.core.features; -import org.vitrivr.cineast.core.features.retriever.CollectionBooleanRetriever; - import java.util.Collections; +import org.vitrivr.cineast.core.features.retriever.CollectionBooleanRetriever; public class DailyCollectionBooleanRetriever extends CollectionBooleanRetriever { - public DailyCollectionBooleanRetriever() { - super("features_daily", Collections.singletonList("location")); - } + + public DailyCollectionBooleanRetriever() { + super("features_daily", Collections.singletonList("location")); + } } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyRangeBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyRangeBooleanRetriever.java index e1fc0255e..dcbe4cc4c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyRangeBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DailyRangeBooleanRetriever.java @@ -3,7 +3,8 @@ import java.util.Arrays; public class DailyRangeBooleanRetriever extends RangeBooleanRetriever { - public DailyRangeBooleanRetriever() { - super("features_daily", Arrays.asList("heart_rate","skim_temp","steps","calories","gsr")); - } + + public DailyRangeBooleanRetriever() { + super("features_daily", Arrays.asList("heart_rate", "skim_temp", "steps", "calories", "gsr")); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantColors.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantColors.java index f4c17e730..0af1b0748 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantColors.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantColors.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; @@ -9,17 +11,14 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.KMeansPP; -import java.util.ArrayList; -import java.util.List; - public class DominantColors extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid16.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid16.java index a2876f84c..24ed9474f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid16.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid16.java @@ -3,22 +3,21 @@ import boofcv.alg.feature.detect.edge.EdgeContour; import boofcv.alg.feature.detect.edge.EdgeSegment; import georegression.struct.point.Point2D_I32; +import java.util.Iterator; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeList; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.Iterator; -import java.util.List; - public class DominantEdgeGrid16 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid8.java index 29e2eec39..69cb740f9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DominantEdgeGrid8.java @@ -3,22 +3,21 @@ import boofcv.alg.feature.detect.edge.EdgeContour; import boofcv.alg.feature.detect.edge.EdgeSegment; import georegression.struct.point.Point2D_I32; +import java.util.Iterator; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeList; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.Iterator; -import java.util.List; - public class DominantEdgeGrid8 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EHD.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EHD.java index 64a82ab3a..3ef126e12 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EHD.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EHD.java @@ -2,6 +2,7 @@ import boofcv.io.image.ConvertBufferedImage; import boofcv.struct.image.GrayU8; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -9,20 +10,16 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.List; - /** - * see Efficient Use of MPEG-7 Edge Histogram Descriptor by Won '02 - * see http://stackoverflow.com/questions/909542/opencv-edge-extraction - * + * see Efficient Use of MPEG-7 Edge Histogram Descriptor by Won '02 see http://stackoverflow.com/questions/909542/opencv-edge-extraction */ public class EHD extends AbstractFeatureModule { @@ -32,7 +29,7 @@ public EHD() { this(80); } - protected EHD(int vectorLength){ + protected EHD(int vectorLength) { super("features_EHD", 16f / 4f, vectorLength); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88.java index 206151596..8078dd8e8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88.java @@ -1,24 +1,23 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeImg; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class EdgeARP88 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88Full.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88Full.java index 37da9279d..e310d325e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88Full.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeARP88Full.java @@ -1,24 +1,23 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeImg; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class EdgeARP88Full extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16.java index f8cde08e9..8e75073e5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16.java @@ -1,24 +1,23 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeImg; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class EdgeGrid16 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16Full.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16Full.java index a04e969df..4c291f760 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16Full.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/EdgeGrid16Full.java @@ -1,24 +1,23 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.descriptor.EdgeImg; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class EdgeGrid16Full extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java index 62257f669..707b10fc1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java @@ -2,6 +2,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -20,10 +23,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.MaskGenerator; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - public class ForegroundBoundingBox extends AbstractFeatureModule { public ForegroundBoundingBox() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java index c5b7b21be..57699e8e7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java @@ -3,6 +3,9 @@ import boofcv.abst.feature.dense.DescribeImageDense; import boofcv.struct.feature.TupleDesc_F64; import boofcv.struct.image.GrayU8; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -16,10 +19,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractCodebookFeatureModule; import org.vitrivr.cineast.core.util.images.HOGHelper; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; - public abstract class HOG extends AbstractCodebookFeatureModule { @@ -82,7 +81,7 @@ protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. * * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. */ @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java index d7ca626e6..1aa61d84a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K256.java @@ -3,24 +3,22 @@ import org.vitrivr.cineast.core.util.MathHelper; /** - * A Extraction and Retrieval module that uses HOG descriptors and a 256 word codebook based on Mirflickr 25K to obtain a - * histograms of codewords. These histograms ares used as feature-vectors. - * + * A Extraction and Retrieval module that uses HOG descriptors and a 256 word codebook based on Mirflickr 25K to obtain a histograms of codewords. These histograms ares used as feature-vectors. */ public class HOGMirflickr25K256 extends HOG { - /** - * Constructor: defines entityname and max distance. - */ - public HOGMirflickr25K256() { - super("features_hogmf25k256", (float)MathHelper.SQRT2/4.0f, 256); - } - /** - * Returns the full name of the codebook to use. All codebooks must be placed in the - * ./resources/codebooks folder. - */ - @Override - protected String codebook() { - return "mirflickr25k-256.hogcb"; - } + /** + * Constructor: defines entityname and max distance. + */ + public HOGMirflickr25K256() { + super("features_hogmf25k256", (float) MathHelper.SQRT2 / 4.0f, 256); + } + + /** + * Returns the full name of the codebook to use. All codebooks must be placed in the ./resources/codebooks folder. + */ + @Override + protected String codebook() { + return "mirflickr25k-256.hogcb"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java index 79b790e4e..9f40fd862 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOGMirflickr25K512.java @@ -3,24 +3,22 @@ import org.vitrivr.cineast.core.util.MathHelper; /** - * A Extraction and Retrieval module that uses HOG descriptors and a 512 word codebook based on Mirflickr 25K to obtain a - * histograms of codewords. These histograms ares used as feature-vectors. - * + * A Extraction and Retrieval module that uses HOG descriptors and a 512 word codebook based on Mirflickr 25K to obtain a histograms of codewords. These histograms ares used as feature-vectors. */ public class HOGMirflickr25K512 extends HOG { - /** - * Constructor: defines name of entity and max distance. - */ - public HOGMirflickr25K512() { - super("features_hogmf25k512", (float)MathHelper.SQRT2/4.0f, 512); - } - /** - * Returns the full name of the codebook to use. All codebooks must be placed in the - * ./resources/codebooks folder. - */ - @Override - protected String codebook() { - return "mirflickr25k-512.hogcb"; - } + /** + * Constructor: defines name of entity and max distance. + */ + public HOGMirflickr25K512() { + super("features_hogmf25k512", (float) MathHelper.SQRT2 / 4.0f, 512); + } + + /** + * Returns the full name of the codebook to use. All codebooks must be placed in the ./resources/codebooks folder. + */ + @Override + protected String codebook() { + return "mirflickr25k-512.hogcb"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12BasslineShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12BasslineShingle.java index 3ff95898f..332e9997b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12BasslineShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12BasslineShingle.java @@ -3,26 +3,22 @@ import org.vitrivr.cineast.core.util.audio.HPCP; /** - * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. The particular module - * focuses on bassline frequencies according to an idea found in [2]. - * - * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. - * IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 - * - * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. - * International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 - * + * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. The particular module focuses on bassline frequencies according to an idea found in [2]. + *

    + * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 + *

    + * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 */ public class HPCP12BasslineShingle extends HPCPShingle { - private final static float MIN_FREQUENCY = 22.0f; + private final static float MIN_FREQUENCY = 22.0f; - private final static float MAX_FREQUENCY = 262.0f; + private final static float MAX_FREQUENCY = 262.0f; - /** - * Default constructor; HPCP Shingle with 12 bins, using frequencies between 5.0f and 262.0f Hz. - */ - public HPCP12BasslineShingle() { - super("features_hpcp12bassline", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); - } + /** + * Default constructor; HPCP Shingle with 12 bins, using frequencies between 5.0f and 262.0f Hz. + */ + public HPCP12BasslineShingle() { + super("features_hpcp12bassline", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12MelodyShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12MelodyShingle.java index 2c071dcd5..6acc80b40 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12MelodyShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12MelodyShingle.java @@ -3,27 +3,23 @@ import org.vitrivr.cineast.core.util.audio.HPCP; /** - * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. The particular module - * focuses on melodic frequencies according to an idea found in [2]. - * - * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. - * IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 - * - * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. - * International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 - * + * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. The particular module focuses on melodic frequencies according to an idea found in [2]. + *

    + * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 + *

    + * [2] Salamon, J., Serrà, J., & Gómez, E. (2012). Tonal representations for music retrieval: from version identification to query-by-humming. International Journal of Multimedia Information Retrieval, 2(1), 45–58. http://doi.org/10.1007/s13735-012-0026-0 */ public class HPCP12MelodyShingle extends HPCPShingle { - private final static float MIN_FREQUENCY = 262.0f; + private final static float MIN_FREQUENCY = 262.0f; - private final static float MAX_FREQUENCY = 5000.0f; + private final static float MAX_FREQUENCY = 5000.0f; - /** - * Default constructor; HPCP Shingle with 12 bins, using frequencies between 262.0f and 5000.0f Hz. - */ - public HPCP12MelodyShingle() { - super("features_hpcp12melody", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); - } + /** + * Default constructor; HPCP Shingle with 12 bins, using frequencies between 262.0f and 5000.0f Hz. + */ + public HPCP12MelodyShingle() { + super("features_hpcp12melody", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12Shingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12Shingle.java index 5a4f5ad37..0685f9c51 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12Shingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCP12Shingle.java @@ -4,14 +4,15 @@ public class HPCP12Shingle extends HPCPShingle { - private final static float MIN_FREQUENCY = 50.0f; - private final static float MAX_FREQUENCY = 5000.0f; + private final static float MIN_FREQUENCY = 50.0f; - /** - * Default constructor; HPCP Shingle with 12 bins, using frequencies between 50.0f and 5000.0f Hz. - */ - public HPCP12Shingle() { - super("features_hpcp12shingle", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); - } + private final static float MAX_FREQUENCY = 5000.0f; + + /** + * Default constructor; HPCP Shingle with 12 bins, using frequencies between 50.0f and 5000.0f Hz. + */ + public HPCP12Shingle() { + super("features_hpcp12shingle", MIN_FREQUENCY, MAX_FREQUENCY, HPCP.Resolution.FULLSEMITONE); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCPShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCPShingle.java index eed02a81a..b0a4b2e35 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCPShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HPCPShingle.java @@ -1,5 +1,9 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -17,200 +21,194 @@ import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; - /** - * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. These shingles can be used - * for cover-song and version identification. - * - * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. - * IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 - * + * An Extraction and Retrieval module that leverages pure HPCP shingles according to [1]. These shingles can be used for cover-song and version identification. + *

    + * [1] Casey, M., Rhodes, C., & Slaney, M. (2008). Analysis of minimum distances in high-dimensional musical spaces. IEEE Transactions on Audio, Speech and Language Processing, 16(5), 1015–1028. http://doi.org/10.1109/TASL.2008.925883 */ public abstract class HPCPShingle extends StagedFeatureModule { - /** Duration of the window during STFT in seconds. */ - private final static float WINDOW_SIZE = 0.200f; - - /** - * Size of a shingle in number of HPCP features. - * - * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components - */ - private final static int SHINGLE_SIZE = 25; - - /** Minimum resolution to consider in HPCP calculation. */ - private final float min_frequency; - - /** Maximum resolution to consider in HPCP calculation. */ - private final float max_frequency; - - /** HPCP resolution (12, 24, 36 bins). */ - private final HPCP.Resolution resolution; - - /** Distance-threshold used to sort out vectors that should should not count in the final scoring stage. */ - private final float distanceThreshold; - - /** - * Default constructor. - * - * @param name Name of the entity (for persistence writer). - * @param min_frequency Minimum frequency to consider during HPCP analysis. - * @param max_frequency Maximum frequency to consider during HPCP analysis. - * @param resolution Resolution of HPCP (i.e. number of HPCP bins). - */ - protected HPCPShingle(String name, float min_frequency, float max_frequency, HPCP.Resolution resolution) { - super(name, 2.0f, SHINGLE_SIZE * resolution.bins); - - /* Assign variables. */ - this.min_frequency = min_frequency; - this.max_frequency = max_frequency; - this.resolution = resolution; - this.distanceThreshold = 0.9f; + /** + * Duration of the window during STFT in seconds. + */ + private final static float WINDOW_SIZE = 0.200f; + + /** + * Size of a shingle in number of HPCP features. + *

    + * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components + */ + private final static int SHINGLE_SIZE = 25; + + /** + * Minimum resolution to consider in HPCP calculation. + */ + private final float min_frequency; + + /** + * Maximum resolution to consider in HPCP calculation. + */ + private final float max_frequency; + + /** + * HPCP resolution (12, 24, 36 bins). + */ + private final HPCP.Resolution resolution; + + /** + * Distance-threshold used to sort out vectors that should should not count in the final scoring stage. + */ + private final float distanceThreshold; + + /** + * Default constructor. + * + * @param name Name of the entity (for persistence writer). + * @param min_frequency Minimum frequency to consider during HPCP analysis. + * @param max_frequency Maximum frequency to consider during HPCP analysis. + * @param resolution Resolution of HPCP (i.e. number of HPCP bins). + */ + protected HPCPShingle(String name, float min_frequency, float max_frequency, HPCP.Resolution resolution) { + super(name, 2.0f, SHINGLE_SIZE * resolution.bins); + + /* Assign variables. */ + this.min_frequency = min_frequency; + this.max_frequency = max_frequency; + this.resolution = resolution; + this.distanceThreshold = 0.9f; + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module. + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + return this.getFeatures(sc).stream().limit(SHINGLE_SIZE * 2).collect(Collectors.toList()); + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Prepare helper data-structures. */ + final HashMap map = new HashMap<>(); + for (DistanceElement hit : partialResults) { + if (hit.getDistance() <= this.distanceThreshold) { + map.merge(hit.getId(), hit, (o, n) -> new SegmentDistanceElement(o.getId(), (o.getDistance() + n.getDistance()) / 2)); + } } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module. - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. + /* Prepare final result-set. */ + final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(map.entrySet().stream().map(e -> e.getValue().toScore(fkt))); + } + + /** + * Processes a SegmentContainer during extraction; calculates the HPCP for the container and creates and persists a set shingle-vectors. + * + * @param segment SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer segment) { + final List features = this.getFeatures(segment); + + /* + * Persists only the individual, disjoint shingle vectors */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - return this.getFeatures(sc).stream().limit(SHINGLE_SIZE * 2).collect(Collectors.toList()); + features.stream() + .filter((feature) -> features.indexOf(feature) % SHINGLE_SIZE == 0) + .forEach((feature) -> this.persist(segment.getId(), new FloatVectorImpl(feature))); + } + + /** + * Returns a modified QueryConfig for the given feature. This implementation copies the original configuaration and sets a weight-vector, which depends on the feature vector. + * + * @param qc Original query config + * @param feature Feature for which a weight-vector is required. + * @return New query config. + */ + protected ReadableQueryConfig queryConfigForFeature(QueryConfig qc, float[] feature) { + float[] weight = new float[feature.length]; + for (int i = 0; i < feature.length; i++) { + if (feature[i] == 0.0f) { + weight[i] = 10.0f; + } } - - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Prepare helper data-structures. */ - final HashMap map = new HashMap<>(); - for (DistanceElement hit : partialResults) { - if (hit.getDistance() <= this.distanceThreshold) { - map.merge(hit.getId(), hit, (o, n) -> new SegmentDistanceElement(o.getId(), (o.getDistance() + n.getDistance())/2)); - } - } - - /* Prepare final result-set. */ - final CorrespondenceFunction fkt = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(map.entrySet().stream().map(e -> e.getValue().toScore(fkt))); - } - - /** - * Processes a SegmentContainer during extraction; calculates the HPCP for the container and - * creates and persists a set shingle-vectors. - * - * @param segment SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer segment) { - final List features = this.getFeatures(segment); - - /* - * Persists only the individual, disjoint shingle vectors - */ - features.stream() - .filter((feature) -> features.indexOf(feature) % SHINGLE_SIZE == 0) - .forEach((feature) -> this.persist(segment.getId(), new FloatVectorImpl(feature))); + return qc.clone().setDistanceWeights(weight); + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean) + .addHint(ReadableQueryConfig.Hints.inexact); + } + + /** + * Returns a list of feature vectors given a SegmentContainer. + * + * @param segment SegmentContainer for which to calculate the feature vectors. + * @return List of HPCP Shingle feature vectors. + */ + private List getFeatures(SegmentContainer segment) { + /* Create STFT; If this fails, return empty list. */ + Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); + STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow()); + if (stft == null) { + return new ArrayList<>(0); } - /** - * Returns a modified QueryConfig for the given feature. This implementation copies the original configuaration and - * sets a weight-vector, which depends on the feature vector. - * - * @param qc Original query config - * @param feature Feature for which a weight-vector is required. - * @return New query config. - */ - protected ReadableQueryConfig queryConfigForFeature(QueryConfig qc, float[] feature) { - float[] weight = new float[feature.length]; - for (int i=0;i getFeatures(SegmentContainer segment) { - /* Create STFT; If this fails, return empty list. */ - Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); - STFT stft = segment.getSTFT(parameters.first,(parameters.first - 2*parameters.second)/2, parameters.second, new HanningWindow()); - if (stft == null) { - return new ArrayList<>(0); - } - - HPCP hpcps = new HPCP(this.resolution, this.min_frequency, this.max_frequency); - hpcps.addContribution(stft); - - int vectors = Math.max(hpcps.size() - SHINGLE_SIZE, 1); - final SummaryStatistics statistics = new SummaryStatistics(); - - List> features = new ArrayList<>(vectors); - for (int n = 0; n < vectors; n++) { - Pair feature = this.getHPCPShingle(hpcps, n); - features.add(feature); - statistics.addValue(feature.first); - } - - final double threshold = 0.25*statistics.getGeometricMean(); - return features.stream().filter(f -> (f.first > threshold)).map(f -> f.second).collect(Collectors.toList()); + List> features = new ArrayList<>(vectors); + for (int n = 0; n < vectors; n++) { + Pair feature = this.getHPCPShingle(hpcps, n); + features.add(feature); + statistics.addValue(feature.first); } - - /** - * Returns a constant length HPCP shingle from a Harmonic Pitch Class Profile. If the - * resulting shingle is a zero-vector, this method will return null instead. - * - * @param hpcps The HPCP to create the shingle from. - * @param shift The index to shift the HPCP's by. - * @return HPCP shingle (feature vector) or null. - */ - private Pair getHPCPShingle(HPCP hpcps, int shift) { - float[] feature = new float[SHINGLE_SIZE * this.resolution.bins]; - for (int i = shift; i < SHINGLE_SIZE + shift; i++) { - if (i < hpcps.size()) { - float[] hpcp = hpcps.getHpcp(i); - System.arraycopy(hpcp, 0, feature, (i - shift) * this.resolution.bins, hpcp.length); - } - } - return new Pair<>(MathHelper.normL2(feature), MathHelper.normalizeL2(feature)); + final double threshold = 0.25 * statistics.getGeometricMean(); + return features.stream().filter(f -> (f.first > threshold)).map(f -> f.second).collect(Collectors.toList()); + } + + + /** + * Returns a constant length HPCP shingle from a Harmonic Pitch Class Profile. If the resulting shingle is a zero-vector, this method will return null instead. + * + * @param hpcps The HPCP to create the shingle from. + * @param shift The index to shift the HPCP's by. + * @return HPCP shingle (feature vector) or null. + */ + private Pair getHPCPShingle(HPCP hpcps, int shift) { + float[] feature = new float[SHINGLE_SIZE * this.resolution.bins]; + for (int i = shift; i < SHINGLE_SIZE + shift; i++) { + if (i < hpcps.size()) { + float[] hpcp = hpcps.getHpcp(i); + System.arraycopy(hpcp, 0, feature, (i - shift) * this.resolution.bins, hpcp.length); + } } + return new Pair<>(MathHelper.normL2(feature), MathHelper.normalizeL2(feature)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueHistogram.java index 4e2aa4d38..77386c9a3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueHistogram.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.HSVContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; @@ -10,8 +11,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class HueHistogram extends AbstractFeatureModule { public HueHistogram() { @@ -26,16 +25,16 @@ public void processSegment(SegmentContainer shot) { float[] hist = new float[16]; - for (VideoFrame frame : shot.getVideoFrames()){ + for (VideoFrame frame : shot.getVideoFrames()) { updateHist(hist, frame.getImage().getThumbnailColors()); } float sum = 0; - for(int i = 0; i < hist.length; ++i){ + for (int i = 0; i < hist.length; ++i) { sum += hist[i]; } - if(sum > 1f){ - for(int i = 0; i < hist.length; ++i){ + if (sum > 1f) { + for (int i = 0; i < hist.length; ++i) { hist[i] /= sum; } } @@ -52,10 +51,10 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc } - private static float[] updateHist(float[] hist, int[] colors){ - for(int color : colors){ + private static float[] updateHist(float[] hist, int[] colors) { + for (int color : colors) { HSVContainer container = ColorConverter.RGBtoHSV(new ReadableRGBContainer(color)); - if(container.getS() > 0.2f && container.getV() > 0.3f){ + if (container.getS() > 0.2f && container.getV() > 0.3f) { float h = container.getH() * hist.length; int idx = (int) h; hist[idx] += h - idx; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueValueVarianceGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueValueVarianceGrid8.java index 1d4f3e99c..258d01fba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueValueVarianceGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HueValueVarianceGrid8.java @@ -1,5 +1,9 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -9,7 +13,10 @@ import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.*; +import org.vitrivr.cineast.core.data.FloatVector; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; @@ -17,11 +24,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - public class HueValueVarianceGrid8 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java index c98cb6af3..679728559 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java @@ -1,6 +1,11 @@ package org.vitrivr.cineast.core.features; import com.twelvemonkeys.image.ImageUtil; +import java.awt.Image; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -15,222 +20,209 @@ import org.vitrivr.cineast.core.render.Renderer; import org.vitrivr.cineast.core.util.LogHelper; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - /** * An abstract base class for light field based feature modules as proposed by [1]. - * - * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). - * On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 - * + *

    + * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 */ public abstract class Lightfield extends StagedFeatureModule { - /** Size of the rendering environment. */ - protected final static int RENDERING_SIZE = 256; - - /** Default value for an unknown pose index. */ - protected final static int POSEIDX_UNKNOWN = -1; + /** + * Size of the rendering environment. + */ + protected final static int RENDERING_SIZE = 256; + + /** + * Default value for an unknown pose index. + */ + protected final static int POSEIDX_UNKNOWN = -1; + + /** + * Camera positions used to create lightfield descriptions. - First index indicates the position-index - Second index can be used to address the x,y and z coordinates. + *

    + * The array must be 1x3 at least, excess elements in the second dimension are being ignored. + */ + private final double[][] camerapositions; + + /** + * Offscreen rendering environment used to create Lightfield images. + */ + private final Renderer renderer; + + protected Lightfield(String tableName, float maxDist, int vectorLength, double[][] camerapositions) { + super(tableName, maxDist, vectorLength); + if (camerapositions.length == 0) { + throw new IllegalArgumentException("You must specify at least one camera position!"); + } + for (double[] position : camerapositions) { + if (position.length < 3) { + throw new IllegalArgumentException("Each position must have at least three coordinates."); + } + } + this.camerapositions = camerapositions; - /** - * Camera positions used to create lightfield descriptions. - * - First index indicates the position-index - * - Second index can be used to address the x,y and z coordinates. - * - * The array must be 1x3 at least, excess elements in the second dimension - * are being ignored. + /* + * Instantiate JOGLOffscreenRenderer. + * Handle the case where it cannot be created due to missing OpenGL support. */ - private final double[][] camerapositions; - - /** Offscreen rendering environment used to create Lightfield images. */ - private final Renderer renderer; - - protected Lightfield(String tableName, float maxDist, int vectorLength, double[][] camerapositions) { - super(tableName, maxDist, vectorLength); - if (camerapositions.length == 0) { - throw new IllegalArgumentException("You must specify at least one camera position!"); - } - for (double[] position : camerapositions) { - if (position.length < 3) { - throw new IllegalArgumentException("Each position must have at least three coordinates."); - } - } - this.camerapositions = camerapositions; - - /* - * Instantiate JOGLOffscreenRenderer. - * Handle the case where it cannot be created due to missing OpenGL support. - */ - JOGLOffscreenRenderer renderer = null; - try { - renderer = new JOGLOffscreenRenderer(RENDERING_SIZE, RENDERING_SIZE); - } catch (Exception exception) { - LOGGER.error("Could not instantiate JOGLOffscreenRenderer! This instance of {} will not create any results or features!", this.getClass().getSimpleName()); - } finally { - this.renderer = renderer; - } + JOGLOffscreenRenderer renderer = null; + try { + renderer = new JOGLOffscreenRenderer(RENDERING_SIZE, RENDERING_SIZE); + } catch (Exception exception) { + LOGGER.error("Could not instantiate JOGLOffscreenRenderer! This instance of {} will not create any results or features!", this.getClass().getSimpleName()); + } finally { + this.renderer = renderer; + } + } + + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + /* Check if renderer could be initialised. */ + if (this.renderer == null) { + LOGGER.error("No renderer found. {} does not return any results.", this.getClass().getSimpleName()); + return new ArrayList<>(0); } + /* Extract features from either the provided Mesh (1) or image (2). */ + ReadableMesh mesh = sc.getNormalizedMesh(); + List features; + if (mesh.isEmpty()) { + BufferedImage image = ImageUtil.createResampled(sc.getAvgImg().getBufferedImage(), RENDERING_SIZE, RENDERING_SIZE, Image.SCALE_SMOOTH); + features = this.featureVectorsFromImage(image, POSEIDX_UNKNOWN); + } else { + features = this.featureVectorsFromMesh(mesh); + } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Check if renderer could be initialised. */ - if (this.renderer == null) { - LOGGER.error("No renderer found. {} does not return any results.", this.getClass().getSimpleName()); - return new ArrayList<>(0); - } - - /* Extract features from either the provided Mesh (1) or image (2). */ - ReadableMesh mesh = sc.getNormalizedMesh(); - List features; - if (mesh.isEmpty()) { - BufferedImage image = ImageUtil.createResampled(sc.getAvgImg().getBufferedImage(), RENDERING_SIZE, RENDERING_SIZE, Image.SCALE_SMOOTH); - features = this.featureVectorsFromImage(image,POSEIDX_UNKNOWN); + return features; + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Perform search for each extracted feature and adjust scores. */ + HashMap map = new HashMap<>(); + for (DistanceElement result : partialResults) { + map.merge(result.getId(), result, (v1, v2) -> { + if (v1.getDistance() < v2.getDistance()) { + return v1; } else { - features = this.featureVectorsFromMesh(mesh); + return v2; } - - return features; + }); } - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Perform search for each extracted feature and adjust scores. */ - HashMap map = new HashMap<>(); - for (DistanceElement result : partialResults) { - map.merge(result.getId(), result, (v1, v2) -> { - if (v1.getDistance() < v2.getDistance()) { - return v1; - } else { - return v2; - } - }); - } - - /* Add results to list and return list of results. */ - final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(map.entrySet().stream().map((e) -> e.getValue().toScore(correspondence))); + /* Add results to list and return list of results. */ + final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(map.entrySet().stream().map((e) -> e.getValue().toScore(correspondence))); + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean); + } + + /** + * Processes a single segment. Extracts the mesh and persists all associated features. Segments that have no mesh or an empty mesh will not be processed. + */ + @Override + public void processSegment(SegmentContainer sc) { + /* Check for renderer. */ + if (this.renderer == null) { + LOGGER.error("No renderer found! {} does not create any features.", this.getClass().getSimpleName()); + return; } - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. - */ - @Override - protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.euclidean); + /* If Mesh is empty, no feature is persisted. */ + ReadableMesh mesh = sc.getNormalizedMesh(); + if (mesh == null || mesh.isEmpty()) { + return; } - /** - * Processes a single segment. Extracts the mesh and persists all associated features. Segments - * that have no mesh or an empty mesh will not be processed. - */ - @Override - public void processSegment(SegmentContainer sc) { - /* Check for renderer. */ - if (this.renderer == null) { - LOGGER.error("No renderer found! {} does not create any features.", this.getClass().getSimpleName()); - return; - } - - /* If Mesh is empty, no feature is persisted. */ - ReadableMesh mesh = sc.getNormalizedMesh(); - if (mesh == null || mesh.isEmpty()) { - return; - } - - /* Extract and persist all features. */ - List features = this.featureVectorsFromMesh(mesh); - for (float[] feature : features) { - this.persist(sc.getId(), new FloatVectorImpl(feature)); - } + /* Extract and persist all features. */ + List features = this.featureVectorsFromMesh(mesh); + for (float[] feature : features) { + this.persist(sc.getId(), new FloatVectorImpl(feature)); + } + } + + /** + * Extracts the Lightfield Fourier descriptors from a provided Mesh. The returned list contains elements of which each holds a pose-index (relative to the camera-positions used by the feature module) and the associated feature-vector (s). + * + * @param mesh Mesh for which to extract the Lightfield Fourier descriptors. + * @return List of descriptors for mesh. + */ + protected List featureVectorsFromMesh(ReadableMesh mesh) { + /* Prepare empty list of features. */ + List features = new ArrayList<>(20); + + /* Retains the renderer and returns if retention fails. */ + if (!this.renderer.retain()) { + return features; } - /** - * Extracts the Lightfield Fourier descriptors from a provided Mesh. The returned list contains - * elements of which each holds a pose-index (relative to the camera-positions used by the feature module) - * and the associated feature-vector (s). - * - * @param mesh Mesh for which to extract the Lightfield Fourier descriptors. - * @return List of descriptors for mesh. + /* Everything happens in the try-catch block so as to make sure, that if any exception occurs, + * the renderer is released again. */ - protected List featureVectorsFromMesh(ReadableMesh mesh) { - /* Prepare empty list of features. */ - List features = new ArrayList<>(20); - - /* Retains the renderer and returns if retention fails. */ - if (!this.renderer.retain()) { + try { + /* Clears the renderer and assembles a new Mesh. */ + this.renderer.clear(); + this.renderer.assemble(mesh); + + /* Obtains rendered image from configured perspective. */ + for (int i = 0; i < this.camerapositions.length; i++) { + /* Adjust the camera and render the image. */ + this.renderer.positionCamera((float) this.camerapositions[i][0], (float) this.camerapositions[i][1], (float) this.camerapositions[i][2]); + this.renderer.render(); + BufferedImage image = this.renderer.obtain(); + if (image == null) { + LOGGER.error("Could not generate feature for {} because no image could be obtained from JOGOffscreenRenderer.", this.getClass().getSimpleName()); return features; } - - /* Everything happens in the try-catch block so as to make sure, that if any exception occurs, - * the renderer is released again. - */ - try { - /* Clears the renderer and assembles a new Mesh. */ - this.renderer.clear(); - this.renderer.assemble(mesh); - - /* Obtains rendered image from configured perspective. */ - for (int i = 0; i < this.camerapositions.length; i++) { - /* Adjust the camera and render the image. */ - this.renderer.positionCamera((float) this.camerapositions[i][0], (float) this.camerapositions[i][1], (float) this.camerapositions[i][2]); - this.renderer.render(); - BufferedImage image = this.renderer.obtain(); - if (image == null) { - LOGGER.error("Could not generate feature for {} because no image could be obtained from JOGOffscreenRenderer.", this.getClass().getSimpleName()); - return features; - } - features.addAll(this.featureVectorsFromImage(image, i)); - } - - } catch (Exception exception) { - LOGGER.error("Could not generate feature for {} because an unknown exception occurred ({}).", this.getClass().getSimpleName(), LogHelper.getStackTrace(exception)); - } finally { - /* Release the rendering context. */ - this.renderer.release(); - } - - /* Extract and persist the feature descriptors. */ - return features; + features.addAll(this.featureVectorsFromImage(image, i)); + } + + } catch (Exception exception) { + LOGGER.error("Could not generate feature for {} because an unknown exception occurred ({}).", this.getClass().getSimpleName(), LogHelper.getStackTrace(exception)); + } finally { + /* Release the rendering context. */ + this.renderer.release(); } - protected abstract List featureVectorsFromImage(BufferedImage image, int poseidx); + /* Extract and persist the feature descriptors. */ + return features; + } - public double[] positionsForPoseidx(int poseidx) { - if (poseidx < this.camerapositions.length) { - return this.camerapositions[poseidx]; - } else { - return null; - } + protected abstract List featureVectorsFromImage(BufferedImage image, int poseidx); + + public double[] positionsForPoseidx(int poseidx) { + if (poseidx < this.camerapositions.length) { + return this.camerapositions[poseidx]; + } else { + return null; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldFourier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldFourier.java index 6ea01b741..744287268 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldFourier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldFourier.java @@ -2,6 +2,9 @@ import boofcv.alg.filter.binary.Contour; import georegression.struct.point.Point2D_I32; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.math3.complex.Complex; import org.apache.commons.math3.transform.DftNormalization; import org.apache.commons.math3.transform.FastFourierTransformer; @@ -12,109 +15,105 @@ import org.vitrivr.cineast.core.util.images.ContourHelper; import org.vitrivr.cineast.core.util.math.MathConstants; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; - /** * An Extraction and Retrieval module for 3D models that leverages Fourier based light field descriptors and as proposed in [1] and [2]. - * - * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). - * On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 - * - * [2] Zhang, D., & Lu, G. (2002). - * An Integrated Approach to Shape Based Image Retrieval. In ACCV2002: The 5th Asian Conference on Computer Vision. Melbourne, Australia. - * + *

    + * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 + *

    + * [2] Zhang, D., & Lu, G. (2002). An Integrated Approach to Shape Based Image Retrieval. In ACCV2002: The 5th Asian Conference on Computer Vision. Melbourne, Australia. */ public class LightfieldFourier extends Lightfield { - /** Size of the feature vector. */ - private static final int SIZE = 128 + 1; /* Number of Coefficients + Pose Idx */ - /** - * Weights used for kNN retrieval based on images / sketches. Higher frequency components (standing for finer details) - * have less weight towards the final result. - * - * Also, the first entry (pose-idx) does count less towards the final distance, if known, and not at all if is unknown. - */ - private static final float[] WEIGHTS_POSE = new float[SIZE]; - private static final float[] WEIGHTS_NOPOSE = new float[SIZE]; - static { - WEIGHTS_POSE[0] = 1.50f; - WEIGHTS_NOPOSE[0] = 0.0f; - for (int i = 1; i< SIZE; i++) { - WEIGHTS_POSE[i] = 1.0f - (i-2)*(1.0f/(2*SIZE)); - WEIGHTS_NOPOSE[i] = 1.0f - (i-2)*(1.0f/(2*SIZE)); - } - } + /** + * Size of the feature vector. + */ + private static final int SIZE = 128 + 1; /* Number of Coefficients + Pose Idx */ - /** Helper class that is used to perform FFT. */ - private final FastFourierTransformer transformer; + /** + * Weights used for kNN retrieval based on images / sketches. Higher frequency components (standing for finer details) have less weight towards the final result. + *

    + * Also, the first entry (pose-idx) does count less towards the final distance, if known, and not at all if is unknown. + */ + private static final float[] WEIGHTS_POSE = new float[SIZE]; + private static final float[] WEIGHTS_NOPOSE = new float[SIZE]; - /** - * Default constructor for LightfieldFourier class. - */ - public LightfieldFourier() { - super("features_lightfieldfourier", 2.0f, SIZE, MathConstants.VERTICES_3D_DODECAHEDRON); - this.transformer = new FastFourierTransformer(DftNormalization.STANDARD); + static { + WEIGHTS_POSE[0] = 1.50f; + WEIGHTS_NOPOSE[0] = 0.0f; + for (int i = 1; i < SIZE; i++) { + WEIGHTS_POSE[i] = 1.0f - (i - 2) * (1.0f / (2 * SIZE)); + WEIGHTS_NOPOSE[i] = 1.0f - (i - 2) * (1.0f / (2 * SIZE)); } + } + + /** + * Helper class that is used to perform FFT. + */ + private final FastFourierTransformer transformer; + /** + * Default constructor for LightfieldFourier class. + */ + public LightfieldFourier() { + super("features_lightfieldfourier", 2.0f, SIZE, MathConstants.VERTICES_3D_DODECAHEDRON); + this.transformer = new FastFourierTransformer(DftNormalization.STANDARD); + } - /** - * Extracts the Lightfield Fourier descriptors from a provided BufferedImage. The returned list contains - * elements for each identified contour of adequate size. - * - * @param image Image for which to extract the Lightfield Fourier descriptors. - * @param poseidx Poseidx of the extracted image. - * @return List of descriptors for image. - */ - @Override - protected List featureVectorsFromImage(BufferedImage image, int poseidx) { - final List contours = ContourHelper.getContours(image); - final List features = new ArrayList<>(); - /* Select the largest, inner contour from the list of available contours. */ - for (Contour contour : contours) { - for (List inner : contour.internal) { - /* Check size of selected contour. */ - if (inner.size() < SIZE * 2) { - continue; - } + /** + * Extracts the Lightfield Fourier descriptors from a provided BufferedImage. The returned list contains elements for each identified contour of adequate size. + * + * @param image Image for which to extract the Lightfield Fourier descriptors. + * @param poseidx Poseidx of the extracted image. + * @return List of descriptors for image. + */ + @Override + protected List featureVectorsFromImage(BufferedImage image, int poseidx) { + final List contours = ContourHelper.getContours(image); + final List features = new ArrayList<>(); - /* Calculate the descriptor for the selected contour. */ - double[] cds = ContourHelper.centroidDistance(inner, true); - Complex[] results = this.transformer.transform(cds, TransformType.FORWARD); - double magnitude = results[0].abs(); - float[] feature = new float[SIZE]; - for (int i = 1; i < SIZE; i++) { - feature[i] = (float) (results[i+1].abs() / magnitude); - } - feature = MathHelper.normalizeL2InPlace(feature); - feature[0] = poseidx; - features.add(feature); - } + /* Select the largest, inner contour from the list of available contours. */ + for (Contour contour : contours) { + for (List inner : contour.internal) { + /* Check size of selected contour. */ + if (inner.size() < SIZE * 2) { + continue; } - return features; + /* Calculate the descriptor for the selected contour. */ + double[] cds = ContourHelper.centroidDistance(inner, true); + Complex[] results = this.transformer.transform(cds, TransformType.FORWARD); + double magnitude = results[0].abs(); + float[] feature = new float[SIZE]; + for (int i = 1; i < SIZE; i++) { + feature[i] = (float) (results[i + 1].abs() / magnitude); + } + feature = MathHelper.normalizeL2InPlace(feature); + feature[0] = poseidx; + features.add(feature); + } } - /** - * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the - * same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. - * - * @param qc Original query config - * @param features List of features for which a QueryConfig is required. - * @return New query config (may be identical to the original one). - */ - @Override - protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { - List configs = new ArrayList<>(features.size()); - for (float[] feature : features) { - if (feature[0] == POSEIDX_UNKNOWN) { - configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_NOPOSE)); - } else { - configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_POSE)); - } - } - return configs; + return features; + } + + /** + * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. + * + * @param qc Original query config + * @param features List of features for which a QueryConfig is required. + * @return New query config (may be identical to the original one). + */ + @Override + protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { + List configs = new ArrayList<>(features.size()); + for (float[] feature : features) { + if (feature[0] == POSEIDX_UNKNOWN) { + configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_NOPOSE)); + } else { + configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_POSE)); + } } + return configs; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldZernike.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldZernike.java index 762327b67..972e3fb84 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldZernike.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LightfieldZernike.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.features; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.math3.complex.Complex; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -8,92 +11,86 @@ import org.vitrivr.cineast.core.util.math.MathConstants; import org.vitrivr.cineast.core.util.math.ZernikeMoments; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; - /** * An Extraction and Retrieval module for 3D models that leverages Zernike moment based light field descriptors and as proposed in [1] and [2]. - * - * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). - * On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 - * - * [2] Zhang, D., & Lu, G. (2002). - * An Integrated Approach to Shape Based Image Retrieval. In ACCV2002: The 5th Asian Conference on Computer Vision. Melbourne, Australia. - * + *

    + * [1] Chen, D.-Y., Tian, X.-P., Shen, Y.-T., & Ouh. (2003). On Visual Similarity Based 3D Model Retrieval. In Eurographics (Vol. 22, pp. 313–318). http://doi.org/KE.2008.4730947 + *

    + * [2] Zhang, D., & Lu, G. (2002). An Integrated Approach to Shape Based Image Retrieval. In ACCV2002: The 5th Asian Conference on Computer Vision. Melbourne, Australia. */ public class LightfieldZernike extends Lightfield { - /** Size of the feature vector. */ - private static final int SIZE = 36 + 1; /* Number of Coefficients + Pose Idx */ - /** - * Default constructor for LightfieldZernike class. - */ - public LightfieldZernike() { - super("features_lightfieldzernike", 2.0f, SIZE, MathConstants.VERTICES_3D_DODECAHEDRON); - } + /** + * Size of the feature vector. + */ + private static final int SIZE = 36 + 1; /* Number of Coefficients + Pose Idx */ + + /** + * Default constructor for LightfieldZernike class. + */ + public LightfieldZernike() { + super("features_lightfieldzernike", 2.0f, SIZE, MathConstants.VERTICES_3D_DODECAHEDRON); + } + + /** + * Weights used for kNN retrieval based on images / sketches. Higher frequency components (standing for finer details) have less weight towards the final result. + *

    + * Also, the first entry (pose-idx) does count less towards the final distance, if known, and not at all if is unknown. + */ + private static final float[] WEIGHTS_POSE = new float[SIZE]; + private static final float[] WEIGHTS_NOPOSE = new float[SIZE]; - /** - * Weights used for kNN retrieval based on images / sketches. Higher frequency components (standing for finer details) - * have less weight towards the final result. - * - * Also, the first entry (pose-idx) does count less towards the final distance, if known, and not at all if is unknown. - */ - private static final float[] WEIGHTS_POSE = new float[SIZE]; - private static final float[] WEIGHTS_NOPOSE = new float[SIZE]; - static { - WEIGHTS_POSE[0] = 1.50f; - WEIGHTS_NOPOSE[0] = 0.0f; - for (int i = 1; i< SIZE; i++) { - WEIGHTS_POSE[i] = 1.0f - (i-2)*(1.0f/(2*SIZE)); - WEIGHTS_NOPOSE[i] = 1.0f - (i-2)*(1.0f/(2*SIZE)); - } + static { + WEIGHTS_POSE[0] = 1.50f; + WEIGHTS_NOPOSE[0] = 0.0f; + for (int i = 1; i < SIZE; i++) { + WEIGHTS_POSE[i] = 1.0f - (i - 2) * (1.0f / (2 * SIZE)); + WEIGHTS_NOPOSE[i] = 1.0f - (i - 2) * (1.0f / (2 * SIZE)); } + } - /** - * Extracts the Lightfield Fourier descriptors from a provided BufferedImage. The returned list contains - * elements for each identified contour of adequate size. - * - * @param image Image for which to extract the Lightfield Fourier descriptors. - * @param poseidx Poseidx of the extracted image. - * @return List of descriptors for image. - */ - @Override - protected List featureVectorsFromImage(BufferedImage image, int poseidx) { - final List moments = ZernikeHelper.zernikeMomentsForShapes(image, RENDERING_SIZE /2, 10); - final List features = new ArrayList<>(moments.size()); - for (ZernikeMoments moment : moments) { - float[] feature = new float[SIZE]; - int i = 0; - for (Complex m : moment.getMoments()) { - feature[i] = (float)m.abs(); - i++; - } - feature = MathHelper.normalizeL2InPlace(feature); - feature[0] = poseidx; - features.add(feature); - } - return features; + /** + * Extracts the Lightfield Fourier descriptors from a provided BufferedImage. The returned list contains elements for each identified contour of adequate size. + * + * @param image Image for which to extract the Lightfield Fourier descriptors. + * @param poseidx Poseidx of the extracted image. + * @return List of descriptors for image. + */ + @Override + protected List featureVectorsFromImage(BufferedImage image, int poseidx) { + final List moments = ZernikeHelper.zernikeMomentsForShapes(image, RENDERING_SIZE / 2, 10); + final List features = new ArrayList<>(moments.size()); + for (ZernikeMoments moment : moments) { + float[] feature = new float[SIZE]; + int i = 0; + for (Complex m : moment.getMoments()) { + feature[i] = (float) m.abs(); + i++; + } + feature = MathHelper.normalizeL2InPlace(feature); + feature[0] = poseidx; + features.add(feature); } + return features; + } - /** - * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the - * same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. - * - * @param qc Original query config - * @param features List of features for which a QueryConfig is required. - * @return New query config (may be identical to the original one). - */ - @Override - protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { - List configs = new ArrayList<>(features.size()); - for (float[] feature : features) { - if (feature[0] == POSEIDX_UNKNOWN) { - configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_NOPOSE)); - } else { - configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_POSE)); - } - } - return configs; + /** + * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. + * + * @param qc Original query config + * @param features List of features for which a QueryConfig is required. + * @return New query config (may be identical to the original one). + */ + @Override + protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { + List configs = new ArrayList<>(features.size()); + for (float[] feature : features) { + if (feature[0] == POSEIDX_UNKNOWN) { + configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_NOPOSE)); + } else { + configs.add(new QueryConfig(qc).setDistanceWeights(WEIGHTS_POSE)); + } } + return configs; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java index cbe9e7b3e..b79f7be72 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MFCCShingle.java @@ -1,6 +1,9 @@ package org.vitrivr.cineast.core.features; import gnu.trove.map.hash.TObjectIntHashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -18,132 +21,129 @@ import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - public class MFCCShingle extends StagedFeatureModule { - /** Size of the window during STFT in # samples. */ - private final static float WINDOW_SIZE = 0.2f; - - /** Size of a single MFCC Shingle. */ - private final static int SHINGLE_SIZE = 25; - - /** Distance-threshold used to sort out vectors that should should not count in the final scoring stage. */ - private final float distanceThreshold; - - /** - * - */ - public MFCCShingle() { - super("features_mfccshingles", 2.0f, SHINGLE_SIZE * 13); - this.distanceThreshold = 0.1f; - } - - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Extract MFCC shingle features from QueryObject. */ - return this.getFeatures(sc).stream().limit(SHINGLE_SIZE * 2).collect(Collectors.toList()); - } - - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* Prepare helper data-structures. */ - final List results = new ArrayList<>(); - final TObjectIntHashMap scoreMap = new TObjectIntHashMap<>(); - - /* Set QueryConfig and extract correspondence function. */ - qc = this.setQueryConfig(qc); - final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); - for (DistanceElement hit : partialResults) { - if (hit.getDistance() < this.distanceThreshold) { - scoreMap.adjustOrPutValue(hit.getId(), 1, scoreMap.get(hit.getId())/2); - } - } - - /* Prepare final result-set. */ - scoreMap.forEachEntry((key, value) -> results.add(new SegmentScoreElement(key, 1.0 - 1.0/value))); - ScoreElement.filterMaximumScores(results.stream()); - return results; + /** + * Size of the window during STFT in # samples. + */ + private final static float WINDOW_SIZE = 0.2f; + + /** + * Size of a single MFCC Shingle. + */ + private final static int SHINGLE_SIZE = 25; + + /** + * Distance-threshold used to sort out vectors that should should not count in the final scoring stage. + */ + private final float distanceThreshold; + + /** + * + */ + public MFCCShingle() { + super("features_mfccshingles", 2.0f, SHINGLE_SIZE * 13); + this.distanceThreshold = 0.1f; + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + /* Extract MFCC shingle features from QueryObject. */ + return this.getFeatures(sc).stream().limit(SHINGLE_SIZE * 2).collect(Collectors.toList()); + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* Prepare helper data-structures. */ + final List results = new ArrayList<>(); + final TObjectIntHashMap scoreMap = new TObjectIntHashMap<>(); + + /* Set QueryConfig and extract correspondence function. */ + qc = this.setQueryConfig(qc); + final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); + for (DistanceElement hit : partialResults) { + if (hit.getDistance() < this.distanceThreshold) { + scoreMap.adjustOrPutValue(hit.getId(), 1, scoreMap.get(hit.getId()) / 2); + } } - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. + /* Prepare final result-set. */ + scoreMap.forEachEntry((key, value) -> results.add(new SegmentScoreElement(key, 1.0 - 1.0 / value))); + ScoreElement.filterMaximumScores(results.stream()); + return results; + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean) + .addHint(ReadableQueryConfig.Hints.inexact); + } + + @Override + public void processSegment(SegmentContainer segment) { + List features = this.getFeatures(segment); + + /* + * Persists only the individual, disjoint shingle vectors */ - @Override - protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.euclidean) - .addHint(ReadableQueryConfig.Hints.inexact); - } - - @Override - public void processSegment(SegmentContainer segment) { - List features = this.getFeatures(segment); - - /* - * Persists only the individual, disjoint shingle vectors - */ - features.stream() - .filter((feature) -> features.indexOf(feature) % SHINGLE_SIZE == 0) - .forEach((feature) -> this.persist(segment.getId(), new FloatVectorImpl(feature))); + features.stream() + .filter((feature) -> features.indexOf(feature) % SHINGLE_SIZE == 0) + .forEach((feature) -> this.persist(segment.getId(), new FloatVectorImpl(feature))); + } + + /** + * Derives and returns a list of MFCC features for a SegmentContainer. + * + * @param segment SegmentContainer to derive the MFCC features from. + * @return List of MFCC Shingles. + */ + private List getFeatures(SegmentContainer segment) { + final Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); + final STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow()); + if (stft == null) { + return new ArrayList<>(0); } - - /** - * Derives and returns a list of MFCC features for a SegmentContainer. - * - * @param segment SegmentContainer to derive the MFCC features from. - * @return List of MFCC Shingles. - */ - private List getFeatures(SegmentContainer segment) { - final Pair parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE); - final STFT stft = segment.getSTFT(parameters.first, (parameters.first-2*parameters.second)/2, parameters.second, new HanningWindow()); - if (stft == null) { - return new ArrayList<>(0); + final List mfccs = MFCC.calculate(stft); + int vectors = mfccs.size() - SHINGLE_SIZE; + + List features = new ArrayList<>(Math.max(1, vectors)); + if (vectors > 0) { + for (int i = 0; i < vectors; i++) { + float[] feature = new float[SHINGLE_SIZE * 13]; + for (int j = 0; j < SHINGLE_SIZE; j++) { + MFCC mfcc = mfccs.get(i + j); + System.arraycopy(mfcc.getCepstra(), 0, feature, 13 * j, 13); } - final List mfccs = MFCC.calculate(stft); - int vectors = mfccs.size() - SHINGLE_SIZE; - - List features = new ArrayList<>(Math.max(1, vectors)); - if (vectors > 0) { - for (int i = 0; i < vectors; i++) { - float[] feature = new float[SHINGLE_SIZE * 13]; - for (int j = 0; j < SHINGLE_SIZE; j++) { - MFCC mfcc = mfccs.get(i + j); - System.arraycopy(mfcc.getCepstra(), 0, feature, 13 * j, 13); - } - if (MathHelper.checkNotZero(feature) && MathHelper.checkNotNaN(feature)) { - features.add(MathHelper.normalizeL2(feature)); - } - } + if (MathHelper.checkNotZero(feature) && MathHelper.checkNotNaN(feature)) { + features.add(MathHelper.normalizeL2(feature)); } - return features; + } } + return features; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColor.java index 2b5f2a5ab..750e114b0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColor.java @@ -1,21 +1,20 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.LabContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.providers.MedianImgProvider; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class MedianColor extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44.java index 5a1f71e6a..770a26974 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44.java @@ -1,21 +1,20 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; -import java.util.List; - public class MedianColorARP44 extends AbstractFeatureModule { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44Normalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44Normalized.java index 9e4de3c23..521140639 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44Normalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorARP44Normalized.java @@ -1,20 +1,19 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ARPartioner; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class MedianColorARP44Normalized extends AbstractFeatureModule { public MedianColorARP44Normalized() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java index 79b399ed3..99a5fb093 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ColorConverter; @@ -7,7 +10,10 @@ import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.*; +import org.vitrivr.cineast.core.data.FloatVector; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; @@ -16,10 +22,6 @@ import org.vitrivr.cineast.core.util.ColorUtils; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - public class MedianColorGrid8 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8Normalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8Normalized.java index acf3522ad..9bbc223fc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8Normalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8Normalized.java @@ -1,18 +1,17 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class MedianColorGrid8Normalized extends MedianColorGrid8 { public MedianColorGrid8Normalized() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java index af33a4034..f3387f706 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java @@ -2,6 +2,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.function.Supplier; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; @@ -9,26 +10,24 @@ import org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import java.util.function.Supplier; - public class MedianColorRaster extends AverageColorRaster { - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - /* TODO: Respect batchSize. */ - this.phandler = supply.get(); - this.phandler.open("features_MedianColorRaster"); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); - } + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + /* TODO: Respect batchSize. */ + this.phandler = supply.get(); + this.phandler.open("features_MedianColorRaster"); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "raster", "hist"); + } + + @Override + MultiImage getMultiImage(SegmentContainer shot) { + return shot.getMedianImg(); + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity("features_MedianColorRaster", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); + } - @Override - MultiImage getMultiImage(SegmentContainer shot){ - return shot.getMedianImg(); - } - - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity("features_MedianColorRaster", true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); - } - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHist.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHist.java index 57c11069f..0b4101872 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHist.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHist.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -13,8 +14,6 @@ import org.vitrivr.cineast.core.extraction.segmenter.FuzzyColorHistogramCalculator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class MedianFuzzyHist extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHistNormalized.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHistNormalized.java index 6d15ec719..2bc163c50 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHistNormalized.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianFuzzyHistNormalized.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; @@ -12,8 +13,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.ImageHistogramEqualizer; -import java.util.List; - public class MedianFuzzyHistNormalized extends AbstractFeatureModule { public MedianFuzzyHistNormalized() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java index 06224f07c..3dcbc2931 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MelodyEstimate.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -20,145 +22,144 @@ import org.vitrivr.cineast.core.util.dsp.filter.frequency.SpectralWhiteningFilter; import org.vitrivr.cineast.core.util.dsp.midi.MidiUtil; -import java.util.ArrayList; -import java.util.List; - public class MelodyEstimate extends StagedFeatureModule { - /** - * Size of a shingle in number of HPCP features. - * - * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components. - */ - private final static int SHINGLE_SIZE = 10; - - /** Size of the window during STFT in seconds. */ - private final static float WINDOW_SIZE = 0.048f; - - /** F0 / Pitch estimator class. */ - private final KLF0PitchEstimator estimator = new KLF0PitchEstimator(); - - /** PitchTracker instance used for pitch-tracking. */ - private final PitchTracker tracker = new PitchTracker(); - - /** - * - */ - public MelodyEstimate() { - super("feature_melodyestimate", 2.0f, SHINGLE_SIZE * 12); + /** + * Size of a shingle in number of HPCP features. + *

    + * The final feature vector has SHINGLE_SIZE * HPCP.Resolution.Bins components. + */ + private final static int SHINGLE_SIZE = 10; + + /** + * Size of the window during STFT in seconds. + */ + private final static float WINDOW_SIZE = 0.048f; + + /** + * F0 / Pitch estimator class. + */ + private final KLF0PitchEstimator estimator = new KLF0PitchEstimator(); + + /** + * PitchTracker instance used for pitch-tracking. + */ + private final PitchTracker tracker = new PitchTracker(); + + /** + * + */ + public MelodyEstimate() { + super("feature_melodyestimate", 2.0f, SHINGLE_SIZE * 12); + } + + /** + * Processes a SegmentContainer for later persisting it in the storage layer. + * + * @param sc The SegmentContainer that should be processed. + */ + @Override + public void processSegment(SegmentContainer sc) { + Melody melody = this.transcribe(sc); + if (melody == null) { + return; } - - /** - * Processes a SegmentContainer for later persisting it in the storage layer. - * - * @param sc The SegmentContainer that should be processed. - */ - @Override - public void processSegment(SegmentContainer sc) { - Melody melody = this.transcribe(sc); - if(melody == null){ - return; - } - List features = this.getFeatures(melody); - for (float[] feature : features) { - this.persist(sc.getId(), new FloatVectorImpl(feature)); - } - } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - Melody melody = this.transcribe(sc); - if(melody == null){ - LOGGER.debug("No melody, skipping"); - return null; - } - return this.getFeatures(melody); - } - - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - /* TODO: Improve... */ - final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(partialResults.stream().map(d -> d.toScore(correspondence))); + List features = this.getFeatures(melody); + for (float[] feature : features) { + this.persist(sc.getId(), new FloatVectorImpl(feature)); } - - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. - */ - @Override - protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.euclidean); + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + Melody melody = this.transcribe(sc); + if (melody == null) { + LOGGER.debug("No melody, skipping"); + return null; } - - /** - * Generates a set of feature vectors for the provided melody and returns them - * - * @param melody Melody for which to generate the vectors. - */ - private List getFeatures(Melody melody) { - - /* TODO: Improve... */ - - /* Prepare empty features array. */ - final int vectors = Math.max((melody.size()-2) - SHINGLE_SIZE, 0); - final List features = new ArrayList<>(vectors); - - /* Preapre feature vectors by averaging 3 adjacents vectors. */ - for (int n = 0; n < vectors; n++) { - float[] hist = new float[SHINGLE_SIZE * 12]; - for (int i=1+n; i<=1+n+SHINGLE_SIZE; i++) { - for (int j=i-1; j<=i+1; j++) { - Pitch pitch = melody.getPitch(j); - int idx = MidiUtil.frequencyToMidi(pitch.getFrequency()) % 12; - hist[(i-n-1) * SHINGLE_SIZE + idx] += pitch.getSalience(); - } - } - if (MathHelper.checkNotZero(hist) && MathHelper.checkNotNaN(hist)) { - features.add(MathHelper.normalizeL2(hist)); - } + return this.getFeatures(melody); + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + /* TODO: Improve... */ + final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(partialResults.stream().map(d -> d.toScore(correspondence))); + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + @Override + protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean); + } + + /** + * Generates a set of feature vectors for the provided melody and returns them + * + * @param melody Melody for which to generate the vectors. + */ + private List getFeatures(Melody melody) { + + /* TODO: Improve... */ + + /* Prepare empty features array. */ + final int vectors = Math.max((melody.size() - 2) - SHINGLE_SIZE, 0); + final List features = new ArrayList<>(vectors); + + /* Preapre feature vectors by averaging 3 adjacents vectors. */ + for (int n = 0; n < vectors; n++) { + float[] hist = new float[SHINGLE_SIZE * 12]; + for (int i = 1 + n; i <= 1 + n + SHINGLE_SIZE; i++) { + for (int j = i - 1; j <= i + 1; j++) { + Pitch pitch = melody.getPitch(j); + int idx = MidiUtil.frequencyToMidi(pitch.getFrequency()) % 12; + hist[(i - n - 1) * SHINGLE_SIZE + idx] += pitch.getSalience(); } - - return features; + } + if (MathHelper.checkNotZero(hist) && MathHelper.checkNotNaN(hist)) { + features.add(MathHelper.normalizeL2(hist)); + } } - private Melody transcribe(SegmentContainer sc) { - /* Calculate STFT and apply spectral whitening. */ - Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); - STFT stft = sc.getSTFT(parameters.first, 0, parameters.second, new HanningWindow()); - stft.applyFilter(new SpectralWhiteningFilter(stft.getWindowsize(), stft.getSamplingrate(), 0.33f, 30)); + return features; + } - float time = stft.timeStepsize(); + private Melody transcribe(SegmentContainer sc) { + /* Calculate STFT and apply spectral whitening. */ + Pair parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE); + STFT stft = sc.getSTFT(parameters.first, 0, parameters.second, new HanningWindow()); + stft.applyFilter(new SpectralWhiteningFilter(stft.getWindowsize(), stft.getSamplingrate(), 0.33f, 30)); - /* Prepare necessary helper data-structures. */ - final List> s = this.estimator.estimatePitch(stft); - this.tracker.initialize(s, time); - this.tracker.trackPitches(); - return this.tracker.extractMelody(10); - } + float time = stft.timeStepsize(); + + /* Prepare necessary helper data-structures. */ + final List> s = this.estimator.estimatePitch(stft); + this.tracker.initialize(s, time); + this.tracker.trackPitches(); + return this.tracker.extractMelody(10); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogram.java index f19debc29..e58c6bad2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogram.java @@ -2,6 +2,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -13,10 +16,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - public class MotionHistogram extends SubDivMotionHistogram { public MotionHistogram() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogramBackground.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogramBackground.java index 4e282162e..cb29ef853 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogramBackground.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MotionHistogramBackground.java @@ -2,6 +2,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -15,10 +18,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - public class MotionHistogramBackground extends SubDivMotionHistogram { public MotionHistogramBackground() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java index e104c76ea..717da2248 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/OCRSearch.java @@ -4,6 +4,14 @@ import boofcv.struct.image.GrayU8; import com.google.common.cache.CacheLoader; import georegression.struct.shapes.Quadrilateral_F64; +import java.awt.AlphaComposite; +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; import org.apache.commons.text.similarity.JaroWinklerSimilarity; import org.opencv.core.CvType; import org.opencv.core.Mat; @@ -12,12 +20,13 @@ import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; -import org.vitrivr.cineast.core.util.*; - -import java.awt.*; -import java.awt.image.BufferedImage; -import java.util.List; -import java.util.*; +import org.vitrivr.cineast.core.util.HungarianAlgorithm; +import org.vitrivr.cineast.core.util.MultiTracker; +import org.vitrivr.cineast.core.util.NeedlemanWunschMerge; +import org.vitrivr.cineast.core.util.TextDetector_EAST; +import org.vitrivr.cineast.core.util.TextRecognizer_CTC; +import org.vitrivr.cineast.core.util.TextStream; +import org.vitrivr.cineast.core.util.ThreadLocalObjectCache; /** * OCR is handled by adding fuzziness / levenshtein-distance support to the query if there are no quotes present (as quotes indicate precision) This makes sense here since we expect small errors from OCR sources @@ -27,25 +36,16 @@ public class OCRSearch extends AbstractTextRetriever { public static final String OCR_TABLE_NAME = "features_ocr"; /** - * Configurations - * rate: - * Rate refers to the rate at which detections are done. E.g.: Rate = 3 --> Detections are done at every third frame - * Increase rate for better inference times - * - * threshold_CIoU: - * CIoU stands for Combined Intersection over Union - * When the CIoU is below 0.4, they are regarded as separate streams - * Should not be changed - * - * threshold_postproc: This is the threshold for the postprocessing stream association step - * Strongly urge not to change - * - * tracker_type: Refers to the tracker which is used - * threshold_stream_length: Refers to the amount of consecutive frames a text should minimally appear in - * If a text appears in less consecutive frames than the threshold, the text is discarded + * Configurations rate: Rate refers to the rate at which detections are done. E.g.: Rate = 3 --> Detections are done at every third frame Increase rate for better inference times + *

    + * threshold_CIoU: CIoU stands for Combined Intersection over Union When the CIoU is below 0.4, they are regarded as separate streams Should not be changed + *

    + * threshold_postproc: This is the threshold for the postprocessing stream association step Strongly urge not to change + *

    + * tracker_type: Refers to the tracker which is used threshold_stream_length: Refers to the amount of consecutive frames a text should minimally appear in If a text appears in less consecutive frames than the threshold, the text is discarded */ private static final int rate = 3; - private static final int batchSize=16; + private static final int batchSize = 16; private static final double threshold_CIoU = 0.4; private static final double threshold_postproc = 8; private static final MultiTracker.TRACKER_TYPE tracker_type = MultiTracker.TRACKER_TYPE.CIRCULANT; @@ -179,7 +179,7 @@ private GrayU8 bufferedImage2GrayU8(SegmentContainer shot, int frame_index) { } /** - * @param streamPast Stream which appears before streamFuture + * @param streamPast Stream which appears before streamFuture * @param streamFuture Stream which appears after streamPast * @return The similarity of the streams by taking into account the spatial distance, the overlap, the text similarity, and the frame offset */ @@ -227,6 +227,7 @@ private void saveText(String id, String recognition) { /** * Computes and returns the IoU of the two rectangular boxes. The method will not work if they are not rectangular + * * @param coordA Coordinate of a rectangular box * @param coordB Coordinate of a rectangular box * @return the intersection over union of the two rectangular coordinates @@ -249,8 +250,8 @@ private double getIntersectionOverUnion(Quadrilateral_F64 coordA, Quadrilateral_ } /** - * getAverageIntersectionOverUnion takes two lists of coordinates and returns the average IoU - * The coordinates provided have to describe a rectangular box + * getAverageIntersectionOverUnion takes two lists of coordinates and returns the average IoU The coordinates provided have to describe a rectangular box + * * @param coordinates1 Sequentially ordered coordinates (from frame i to frame i+n) * @param coordinates2 Sequentially ordered coordinates (from frame i to frame i+n) * @return The average intersection over union @@ -301,7 +302,7 @@ public void processSegment(SegmentContainer shot) { matFrames.add(img2Mat(shot.getVideoFrames().get(i).getImage().getBufferedImage())); } - List detections = detector.detect(matFrames, batchSize); + List detections = detector.detect(matFrames, batchSize); List frames_grayU8 = new ArrayList<>(); for (int i = 0; i < lenVideo; i++) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ObjectInstances.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ObjectInstances.java index bd483a033..772afdace 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ObjectInstances.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ObjectInstances.java @@ -1,6 +1,5 @@ package org.vitrivr.cineast.core.features; -import java.util.Collections; import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.score.ScoreElement; @@ -21,7 +20,7 @@ public ObjectInstances() { @Override public void processSegment(SegmentContainer shot) { //TODO implement extraction - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException(); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java index 0d7bf48e6..070335f04 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java @@ -1,12 +1,16 @@ package org.vitrivr.cineast.core.features; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveProviderComparator; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.features.abstracts.BooleanRetriever; -import java.util.*; - public class RangeBooleanRetriever extends BooleanRetriever { private static final List SUPPORTED_OPERATORS = @@ -37,35 +41,35 @@ protected Collection getSupportedOperators() { return SUPPORTED_OPERATORS; } - public PrimitiveTypeProvider getMinimum(String column){ - if (this.attributes.contains(column) && !this.minimumMap.containsKey(column)){ + public PrimitiveTypeProvider getMinimum(String column) { + if (this.attributes.contains(column) && !this.minimumMap.containsKey(column)) { populateExtremaMap(); } return minimumMap.get(column); } - public PrimitiveTypeProvider getMaximum(String column){ - if (this.attributes.contains(column) && !this.maximumMap.containsKey(column)){ + public PrimitiveTypeProvider getMaximum(String column) { + if (this.attributes.contains(column) && !this.maximumMap.containsKey(column)) { populateExtremaMap(); } return maximumMap.get(column); } - private void populateExtremaMap(){ + private void populateExtremaMap() { PrimitiveProviderComparator comparator = new PrimitiveProviderComparator(); - for(String column: this.attributes){ + for (String column : this.attributes) { List col = this.selector.getAll(column); - if (col.isEmpty()){ + if (col.isEmpty()) { continue; } PrimitiveTypeProvider min = col.get(0), max = col.get(0); - for (PrimitiveTypeProvider t : col){ - if (comparator.compare(t, min) < 0){ + for (PrimitiveTypeProvider t : col) { + if (comparator.compare(t, min) < 0) { min = t; } - if(comparator.compare(t, max) > 0){ + if (comparator.compare(t, max) > 0) { max = t; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java index 0999ce1c5..b6f78010e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java @@ -1,21 +1,20 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import java.util.List; - public class STMP7EH extends EHD { private static final Logger LOGGER = LogManager.getLogger(); - public STMP7EH(){ + public STMP7EH() { super(160); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java index 6f819cb6d..962e58e69 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java @@ -3,6 +3,8 @@ import boofcv.abst.feature.detdesc.DetectDescribePoint; import boofcv.struct.feature.BrightFeature; import boofcv.struct.image.GrayF32; +import java.util.ArrayList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -16,86 +18,79 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractCodebookFeatureModule; import org.vitrivr.cineast.core.util.images.SURFHelper; -import java.util.ArrayList; -import java.util.List; - public abstract class SURF extends AbstractCodebookFeatureModule { - private static final Logger LOGGER = LogManager.getLogger(); + + private static final Logger LOGGER = LogManager.getLogger(); - private static QueryConfig.Distance DEFAULT_DISTANCE = QueryConfig.Distance.chisquared; + private static QueryConfig.Distance DEFAULT_DISTANCE = QueryConfig.Distance.chisquared; - protected SURF(String tableName, int vectorLength) { + protected SURF(String tableName, int vectorLength) { super(tableName, 2.0f, vectorLength); - } + } - @Override - public void processSegment(SegmentContainer shot) { - if (shot.getMostRepresentativeFrame() == VideoFrame.EMPTY_VIDEO_FRAME) { - return; - } - DetectDescribePoint descriptors = SURFHelper.getStableSurf(shot.getMostRepresentativeFrame().getImage().getBufferedImage()); - if (descriptors != null && descriptors.getNumberOfFeatures() > 0) { - float[] histogram_f = this.histogram(true, descriptors); - this.persist(shot.getId(), new FloatVectorImpl(histogram_f)); - } else { - LOGGER.warn("No SURF feature could be extracted for segment {}. This is not necessarily an error!", shot.getId()); - } + @Override + public void processSegment(SegmentContainer shot) { + if (shot.getMostRepresentativeFrame() == VideoFrame.EMPTY_VIDEO_FRAME) { + return; } + DetectDescribePoint descriptors = SURFHelper.getStableSurf(shot.getMostRepresentativeFrame().getImage().getBufferedImage()); + if (descriptors != null && descriptors.getNumberOfFeatures() > 0) { + float[] histogram_f = this.histogram(true, descriptors); + this.persist(shot.getId(), new FloatVectorImpl(histogram_f)); + } else { + LOGGER.warn("No SURF feature could be extracted for segment {}. This is not necessarily an error!", shot.getId()); + } + } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - *

    - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module. - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return List of feature vectors for lookup. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Prepare feature pair. */ - List features = new ArrayList<>(1); - - /* Extract features. */ - DetectDescribePoint descriptors = SURFHelper.getStableSurf(sc.getAvgImg().getBufferedImage()); - if (descriptors != null && descriptors.getNumberOfFeatures() > 0) { - features.add(this.histogram(true, descriptors)); - } + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + *

    + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module. + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return List of feature vectors for lookup. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + /* Prepare feature pair. */ + List features = new ArrayList<>(1); - return features; + /* Extract features. */ + DetectDescribePoint descriptors = SURFHelper.getStableSurf(sc.getAvgImg().getBufferedImage()); + if (descriptors != null && descriptors.getNumberOfFeatures() > 0) { + features.add(this.histogram(true, descriptors)); } - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - final CorrespondenceFunction function = qc.getCorrespondenceFunction().orElse( - correspondence); - return ScoreElement.filterMaximumScores(partialResults.stream().map(r -> r.toScore(function))); - } + return features; + } - /** - * Merges the provided QueryConfig with the default QueryConfig enforced by the - * feature module. - * - * @param qc QueryConfig provided by the caller of the feature module. - * @return Modified QueryConfig. - */ - protected ReadableQueryConfig queryConfig(ReadableQueryConfig qc, float[] weights) { - return new QueryConfig(qc) - .setCorrespondenceFunctionIfEmpty(this.correspondence) - .setDistanceIfEmpty(QueryConfig.Distance.chisquared) - .setDistanceWeights(weights); - } + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + final CorrespondenceFunction function = qc.getCorrespondenceFunction().orElse( + correspondence); + return ScoreElement.filterMaximumScores(partialResults.stream().map(r -> r.toScore(function))); + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + protected ReadableQueryConfig queryConfig(ReadableQueryConfig qc, float[] weights) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.chisquared) + .setDistanceWeights(weights); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java index 13296302a..54c40693d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K256.java @@ -1,23 +1,22 @@ package org.vitrivr.cineast.core.features; /** - * A Extraction and Retrieval module that uses SURF descriptors and a 256 word codebook based on Mirflickr 25K to obtain a - * histograms of codewords. These histograms ares used as feature-vectors. - * + * A Extraction and Retrieval module that uses SURF descriptors and a 256 word codebook based on Mirflickr 25K to obtain a histograms of codewords. These histograms ares used as feature-vectors. */ public class SURFMirflickr25K256 extends SURF { - /** - * Default constructor. - */ - public SURFMirflickr25K256() { - super("features_surfmf25k256", 256); - } - /** - * Returns the full name of the Codebook to use. - */ - @Override - protected String codebook() { - return "mirflickr25k-256.surfcb"; - } + /** + * Default constructor. + */ + public SURFMirflickr25K256() { + super("features_surfmf25k256", 256); + } + + /** + * Returns the full name of the Codebook to use. + */ + @Override + protected String codebook() { + return "mirflickr25k-256.surfcb"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java index c41c06fde..5ad361022 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURFMirflickr25K512.java @@ -1,23 +1,22 @@ package org.vitrivr.cineast.core.features; /** - * A Extraction and Retrieval module that uses SURF descriptors and a 512 word codebook based on Mirflickr 25K to obtain a - * histograms of codewords. These histograms ares used as feature-vectors. - * + * A Extraction and Retrieval module that uses SURF descriptors and a 512 word codebook based on Mirflickr 25K to obtain a histograms of codewords. These histograms ares used as feature-vectors. */ public class SURFMirflickr25K512 extends SURF { - /** - * Default constructor. - */ - public SURFMirflickr25K512() { - super("features_surfmf25k512", 512); - } - /** - * Returns the full name of the Codebook to use. - */ - @Override - protected String codebook() { - return "mirflickr25k-512.surfcb"; - } + /** + * Default constructor. + */ + public SURFMirflickr25K512() { + super("features_surfmf25k512", 512); + } + + /** + * Returns the full name of the Codebook to use. + */ + @Override + protected String codebook() { + return "mirflickr25k-512.surfcb"; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SaturationGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SaturationGrid8.java index 7d2c642da..6a52f96d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SaturationGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SaturationGrid8.java @@ -1,5 +1,9 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,7 +11,10 @@ import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.*; +import org.vitrivr.cineast.core.data.FloatVector; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; @@ -15,11 +22,6 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.GridPartitioner; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - public class SaturationGrid8 extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java index b9ba2eeb8..524e97ef5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ShapeCentroidDistance.java @@ -2,6 +2,9 @@ import boofcv.alg.filter.binary.Contour; import georegression.struct.point.Point2D_I32; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.math3.complex.Complex; import org.apache.commons.math3.transform.DftNormalization; import org.apache.commons.math3.transform.FastFourierTransformer; @@ -14,69 +17,65 @@ import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.images.ContourHelper; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; - public class ShapeCentroidDistance extends AbstractFeatureModule { - private static final int DESCRIPTOR_LENGTH = 100; + private static final int DESCRIPTOR_LENGTH = 100; - /** - * - */ - public ShapeCentroidDistance() { - super("features_shapecentroid", 2.0f, DESCRIPTOR_LENGTH); - } + /** + * + */ + public ShapeCentroidDistance() { + super("features_shapecentroid", 2.0f, DESCRIPTOR_LENGTH); + } - @Override - public void processSegment(SegmentContainer shot) { + @Override + public void processSegment(SegmentContainer shot) { - BufferedImage image = shot.getAvgImg().getBufferedImage(); + BufferedImage image = shot.getAvgImg().getBufferedImage(); - List contours = ContourHelper.getContours(image); - List contour = contours.get(0).internal.get(0); + List contours = ContourHelper.getContours(image); + List contour = contours.get(0).internal.get(0); - if (image != null) { - FastFourierTransformer transformer = new FastFourierTransformer(DftNormalization.STANDARD); - double[] distancefunction = ContourHelper.centroidDistance(contour, true); - Complex[] signature = transformer.transform(distancefunction, TransformType.FORWARD); - float[] descriptors = new float[DESCRIPTOR_LENGTH]; - for (int i = 1;i getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - BufferedImage image = sc.getAvgImg().getBufferedImage(); + BufferedImage image = sc.getAvgImg().getBufferedImage(); - qc = setQueryConfig(qc); + qc = setQueryConfig(qc); - List contours = ContourHelper.getContours(image); - List contour = contours.get(0).internal.get(0); + List contours = ContourHelper.getContours(image); + List contour = contours.get(0).internal.get(0); - if (image != null) { - FastFourierTransformer transformer = new FastFourierTransformer(DftNormalization.STANDARD); - double[] distancefunction = ContourHelper.centroidDistance(contour, true); - Complex[] signature = transformer.transform(distancefunction, TransformType.FORWARD); - float[] descriptors = new float[DESCRIPTOR_LENGTH]; - for (int i = 1;i(); - } - } - - @Override - protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc).setDistanceIfEmpty(QueryConfig.Distance.euclidean); + if (image != null) { + FastFourierTransformer transformer = new FastFourierTransformer(DftNormalization.STANDARD); + double[] distancefunction = ContourHelper.centroidDistance(contour, true); + Complex[] signature = transformer.transform(distancefunction, TransformType.FORWARD); + float[] descriptors = new float[DESCRIPTOR_LENGTH]; + for (int i = 1; i < DESCRIPTOR_LENGTH; i++) { + descriptors[i] = (float) (signature[i].abs() / signature[0].abs()); + } + return this.getSimilar(descriptors, qc); + } else { + return new ArrayList<>(); } - + } + + @Override + protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc).setDistanceIfEmpty(QueryConfig.Distance.euclidean); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SpatialDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SpatialDistance.java index 821d03422..b7d70074b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SpatialDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SpatialDistance.java @@ -18,21 +18,18 @@ import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; import org.vitrivr.cineast.core.data.query.containers.ParameterisedLocationQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.ParameterisedLocationQueryTermContainer.ParameterisedLocation; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MetadataFeatureModule; /** - * A feature that calculates similarity based on an approximation of the great-circle distance - * between two objects. + * A feature that calculates similarity based on an approximation of the great-circle distance between two objects. * *

    It extracts the latitude and longitude coordinates provided by Exif GPS data, if available, * or a supplementary JSON file. See {@link GpsData} for more information. * *

    During retrieval, it does similarity search using the Haversine distance as an - * approximation of the great-circle distance. As of now, the feature uses a linear correspondence - * function with a maximum distance of {@code 1000m}. This correspondence + * approximation of the great-circle distance. As of now, the feature uses a linear correspondence function with a maximum distance of {@code 1000m}. This correspondence * heavily influences the quality of the retrieval and is likely to be unfit for some data * sets. */ @@ -88,10 +85,7 @@ public Optional extractFeature(SegmentContainer segmentContainer) { } /** - * Extracts the latitude and longitude coordinates from the specified path using Exif GPS data. If - * not present, it retrieves the coordinates from a complementary JSON file named after the - * original document, e.g. {@code image_0001.json} for {@code image_0001.jpg}, by parsing the - * {@code "latitude"} and {@code "longitude"} key-value pairs. For example: + * Extracts the latitude and longitude coordinates from the specified path using Exif GPS data. If not present, it retrieves the coordinates from a complementary JSON file named after the original document, e.g. {@code image_0001.json} for {@code image_0001.jpg}, by parsing the {@code "latitude"} and {@code "longitude"} key-value pairs. For example: * * 

        *   {
@@ -102,8 +96,7 @@ public Optional extractFeature(SegmentContainer segmentContainer) {
    *   }
    * * @param objectPath Path to the file for which the coordinates should be extracted. - * @return an {@link Optional} containing a {@link Location} based on the extracted Exif or JSON - * coordinates, if found, otherwise an empty {@code Optional}. + * @return an {@link Optional} containing a {@link Location} based on the extracted Exif or JSON coordinates, if found, otherwise an empty {@code Optional}. */ @Override public Optional extractFeature(String objectId, Path objectPath) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java index aa1858139..aeb034527 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonics.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.math3.complex.Complex; import org.apache.commons.math3.util.FastMath; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -15,186 +17,174 @@ import org.vitrivr.cineast.core.util.MathHelper; import org.vitrivr.cineast.core.util.math.functions.SphericalHarmonicsFunction; -import java.util.ArrayList; -import java.util.List; - /** * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. - * - * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). - * A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 - - */ + *

    + * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 + */ public abstract class SphericalHarmonics extends StagedFeatureModule { - /** Voxelizer instance used with thes feature module. */ - private final Voxelizer voxelizer; - /* Size of the Voxel Grid in each of the three dimensions. */ - private final int grid_size; - - /* The maximum harmonic to consider for the feature vector. */ - private final int min_l; - - /* The maximum harmonic to consider for the feature vector. */ - private final int max_l; - - /** - * Constructor for SphericalHarmonics feature module. - * - * @param name Name of the entity for storage. - * @param grid_size Size of the Voxel-Grid - * @param max_l Maximum harmonic l to consider for feature vector. - */ - public SphericalHarmonics(String name, int grid_size, int min_l, int max_l) { - super(name, 2.0f, (grid_size/2 - 10)*(SphericalHarmonicsFunction.numberOfCoefficients(max_l, true) - SphericalHarmonicsFunction.numberOfCoefficients(min_l-1, true))); - this.grid_size = grid_size; - this.min_l = min_l; - this.max_l = max_l; - this.voxelizer = new Voxelizer(2.0f/grid_size); + /** + * Voxelizer instance used with thes feature module. + */ + private final Voxelizer voxelizer; + + /* Size of the Voxel Grid in each of the three dimensions. */ + private final int grid_size; + + /* The maximum harmonic to consider for the feature vector. */ + private final int min_l; + + /* The maximum harmonic to consider for the feature vector. */ + private final int max_l; + + /** + * Constructor for SphericalHarmonics feature module. + * + * @param name Name of the entity for storage. + * @param grid_size Size of the Voxel-Grid + * @param max_l Maximum harmonic l to consider for feature vector. + */ + public SphericalHarmonics(String name, int grid_size, int min_l, int max_l) { + super(name, 2.0f, (grid_size / 2 - 10) * (SphericalHarmonicsFunction.numberOfCoefficients(max_l, true) - SphericalHarmonicsFunction.numberOfCoefficients(min_l - 1, true))); + this.grid_size = grid_size; + this.min_l = min_l; + this.max_l = max_l; + this.voxelizer = new Voxelizer(2.0f / grid_size); + } + + @Override + public void processSegment(SegmentContainer shot) { + /* Get the normalized Mesh. */ + ReadableMesh mesh = shot.getNormalizedMesh(); + if (mesh == null || mesh.isEmpty()) { + return; } - @Override - public void processSegment(SegmentContainer shot) { - /* Get the normalized Mesh. */ - ReadableMesh mesh = shot.getNormalizedMesh(); - if (mesh == null || mesh.isEmpty()) { - return; - } - - /* Extract feature and persist it. */ - float[] feature = this.featureVectorFromMesh(mesh); - this.persist(shot.getId(), new FloatVectorImpl(feature)); + /* Extract feature and persist it. */ + float[] feature = this.featureVectorFromMesh(mesh); + this.persist(shot.getId(), new FloatVectorImpl(feature)); + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + * + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module. + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return List of feature vectors for lookup. + */ + @Override + protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { + /* Initialize list of features. */ + List features = new ArrayList<>(); + + /* Get the normalized Mesh. */ + ReadableMesh mesh = sc.getNormalizedMesh(); + if (mesh == null || mesh.isEmpty()) { + return features; } - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - * - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module. - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return List of feature vectors for lookup. - */ - @Override - protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Initialize list of features. */ - List features = new ArrayList<>(); - - /* Get the normalized Mesh. */ - ReadableMesh mesh = sc.getNormalizedMesh(); - if (mesh == null || mesh.isEmpty()) { - return features; - } - - /* Extract feature and persist it. */ - features.add(this.featureVectorFromMesh(mesh)); - return features; + /* Extract feature and persist it. */ + features.add(this.featureVectorFromMesh(mesh)); + return features; + } + + /** + * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval. + * + * @param partialResults List of partial results returned by the lookup stage. + * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. + * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. + */ + @Override + protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { + final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); + return ScoreElement.filterMaximumScores(partialResults.stream().map(v -> v.toScore(correspondence))); + } + + /** + * Obtains the SphericalHarmonic descriptor of the Mesh. To do so, the Mesh is rasterized into a VoxelGrid of 65 x 65 x 65 Voxels (one *safety-voxel* per dimension to prevent ArrayIndexOutOfBounds exceptions) + *

    + * This VoxelGrid is treated as a function f(x,y,z) = 1.0 if Voxel is visible and 0.0 otherwise. The grid is sampled at 7 different radii r ranging from 0.25 to 1.0, where 0.0 lies at the center of the grid and 1.0 touches the bounding-box of the grid. The parameters r, ϑ, ϼ relate to the VoxelGrid as follows: + *

    + * f(x,y,z) = f(r * sin(ϑ) * cos(ϼ) * α + α, r * cos(ϑ) * α, + α, r * sin(ϑ) * sin(ϼ) * α + α) + *

    + * Where α is a constant used to translate the normalized coordinate system to the bounds of the VoxelGrid. + *

    + * Now for l = 0 to l = 4 (m = -l to +l), the projection of the function f(x,y,z) onto the SphericalHarmonic function Zlm (i.e. the integral ∫f(ϑ,ϼ)Zlm(ϑ,ϼ)dϴdϑ) is calculated. This is done for all of the seven radii. This yields 25 descriptors per radius which results in a feature vector of 7 * 25 entries. + *

    + * Depending on the model, the first components may be 0.0 because the surface of the sphere defined by the radius only touches empty space (i.e the hollow interior of the model). + */ + private float[] featureVectorFromMesh(ReadableMesh mesh) { + final float increment = 0.1f; /* Increment of the angles during calculation of the descriptors. */ + final int cap = 10; /* Cap on R (i.e. radii up to R-cap are considered). */ + final int R = this.grid_size / 2; + final int numberOfCoefficients = SphericalHarmonicsFunction.numberOfCoefficients(this.max_l, true) - SphericalHarmonicsFunction.numberOfCoefficients(this.min_l - 1, true); + + /* Prepares an empty array for the feature vector. */ + float[] feature = new float[(R - cap) * numberOfCoefficients]; + + /* Voxelizes the grid from the mesh. If the resulting grid is invisible, the method returns immediately. */ + VoxelGrid grid = this.voxelizer.voxelize(mesh, this.grid_size + 1, this.grid_size + 1, this.grid_size + 1); + if (!grid.isVisible()) { + return feature; } - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @param qc A ReadableQueryConfig object that contains query-related configuration parameters. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - @Override - protected List postprocessQuery(List partialResults, ReadableQueryConfig qc) { - final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence); - return ScoreElement.filterMaximumScores(partialResults.stream().map(v -> v.toScore(correspondence))); - } + List> descriptors = new ArrayList<>(); - /** - * Obtains the SphericalHarmonic descriptor of the Mesh. To do so, the Mesh is rasterized into a VoxelGrid of 65 x 65 x 65 - * Voxels (one *safety-voxel* per dimension to prevent ArrayIndexOutOfBounds exceptions) - * - * This VoxelGrid is treated as a function f(x,y,z) = 1.0 if Voxel is visible and 0.0 otherwise. The grid is - * sampled at 7 different radii r ranging from 0.25 to 1.0, where 0.0 lies at the center of the grid and 1.0 touches - * the bounding-box of the grid. The parameters r, ϑ, ϼ relate to the VoxelGrid as follows: - * - * f(x,y,z) = f(r * sin(ϑ) * cos(ϼ) * α + α, r * cos(ϑ) * α, + α, r * sin(ϑ) * sin(ϼ) * α + α) - * - * Where α is a constant used to translate the normalized coordinate system to the bounds of the VoxelGrid. - * - * Now for l = 0 to l = 4 (m = -l to +l), the projection of the function f(x,y,z) onto the SphericalHarmonic function Zlm - * (i.e. the integral ∫f(ϑ,ϼ)Zlm(ϑ,ϼ)dϴdϑ) is calculated. This is done for all of the seven radii. This yields 25 descriptors per radius which results in a - * feature vector of 7 * 25 entries. - * - * Depending on the model, the first components may be 0.0 because the surface of the sphere defined by the radius only - * touches empty space (i.e the hollow interior of the model). + /* + * Outer-loops; iterate from l=0 to 5 and m=-l to +l. For each combination, a new SphericalHarmonicsFunction is + * created. */ - private float[] featureVectorFromMesh(ReadableMesh mesh) { - final float increment = 0.1f; /* Increment of the angles during calculation of the descriptors. */ - final int cap = 10; /* Cap on R (i.e. radii up to R-cap are considered). */ - final int R = this.grid_size/2; - final int numberOfCoefficients = SphericalHarmonicsFunction.numberOfCoefficients(this.max_l, true) - SphericalHarmonicsFunction.numberOfCoefficients(this.min_l-1, true); - - /* Prepares an empty array for the feature vector. */ - float[] feature = new float[(R-cap)*numberOfCoefficients]; - - /* Voxelizes the grid from the mesh. If the resulting grid is invisible, the method returns immediately. */ - VoxelGrid grid = this.voxelizer.voxelize(mesh, this.grid_size + 1, this.grid_size + 1, this.grid_size + 1); - if (!grid.isVisible()) { - return feature; - } + for (int l = this.min_l; l <= this.max_l; l++) { + for (int m = 0; m <= l; m++) { - List> descriptors = new ArrayList<>(); + final SphericalHarmonicsFunction fkt = new SphericalHarmonicsFunction(l, m); /* - * Outer-loops; iterate from l=0 to 5 and m=-l to +l. For each combination, a new SphericalHarmonicsFunction is - * created. + * Middle-loop; Iterate over the 7 radii. */ - for (int l = this.min_l; l<=this.max_l; l++) { - for (int m = 0; m <= l; m++) { - - final SphericalHarmonicsFunction fkt = new SphericalHarmonicsFunction(l,m); - - /* - * Middle-loop; Iterate over the 7 radii. - */ - for (int r=0;r()); - } - List list = descriptors.get(r); - - Complex result = new Complex(0.0); - - /* - * Used to calculate the projections at radius r for l and m (i.e. the integral ∫f(ϑ,ϼ)Zlm(ϑ,ϼ)dϴdϑ) - */ - for (float theta=0.0f; theta<=2*Math.PI;theta+=increment) { - for (float phi=0.0f; phi<=Math.PI;phi+=increment) { - int x = (int)((r+1) * FastMath.sin(theta) * FastMath.cos(phi)) + R; - int y = (int)((r+1) * FastMath.cos(theta)) + R; - int z = (int)((r+1) * FastMath.sin(theta) * FastMath.sin(phi)) + R; - - if (grid.isVisible(x,y,z)) { - result = result.add(fkt.value(theta, phi).conjugate().multiply(increment*increment)); - } - } - } - - list.add(result); - } + for (int r = 0; r < R - cap; r++) { + /* Allocate array list for radius. */ + if (descriptors.size() <= r) { + descriptors.add(new ArrayList<>()); + } + List list = descriptors.get(r); + + Complex result = new Complex(0.0); + + /* + * Used to calculate the projections at radius r for l and m (i.e. the integral ∫f(ϑ,ϼ)Zlm(ϑ,ϼ)dϴdϑ) + */ + for (float theta = 0.0f; theta <= 2 * Math.PI; theta += increment) { + for (float phi = 0.0f; phi <= Math.PI; phi += increment) { + int x = (int) ((r + 1) * FastMath.sin(theta) * FastMath.cos(phi)) + R; + int y = (int) ((r + 1) * FastMath.cos(theta)) + R; + int z = (int) ((r + 1) * FastMath.sin(theta) * FastMath.sin(phi)) + R; + + if (grid.isVisible(x, y, z)) { + result = result.add(fkt.value(theta, phi).conjugate().multiply(increment * increment)); + } } - } + } - /* Assembles the actual feature vector. */ - int i = 0; - for (List radius : descriptors) { - for (Complex descriptor : radius) { - feature[i] = (float)descriptor.abs(); - i++; - } + list.add(result); } + } + } - /* Returns the normalized vector. */ - return MathHelper.normalizeL2(feature); + /* Assembles the actual feature vector. */ + int i = 0; + for (List radius : descriptors) { + for (Complex descriptor : radius) { + feature[i] = (float) descriptor.abs(); + i++; + } } + + /* Returns the normalized vector. */ + return MathHelper.normalizeL2(feature); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsDefault.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsDefault.java index 53560b5f6..86846a58e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsDefault.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsDefault.java @@ -1,19 +1,16 @@ package org.vitrivr.cineast.core.features; /** - * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version - * uses mid-resolution settings in terms of harmonics to consider. - * - * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). - * A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 - - */ + * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version uses mid-resolution settings in terms of harmonics to consider. + *

    + * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 + */ public class SphericalHarmonicsDefault extends SphericalHarmonics { - /** - * Constructor for SphericalHarmonics feature module. Uses the values - * for grid_size proposed in [1] and harmonics up to l=6 - */ - public SphericalHarmonicsDefault() { - super("features_sphericalhdefault", 64, 0, 4); - } + + /** + * Constructor for SphericalHarmonics feature module. Uses the values for grid_size proposed in [1] and harmonics up to l=6 + */ + public SphericalHarmonicsDefault() { + super("features_sphericalhdefault", 64, 0, 4); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsHigh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsHigh.java index 862cfbd04..519c01f9e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsHigh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsHigh.java @@ -1,19 +1,16 @@ package org.vitrivr.cineast.core.features; /** - * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version - * uses high-resolution settings in terms of voxel-grid size and harmonics to use. - * - * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). - * A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 - - */ + * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version uses high-resolution settings in terms of voxel-grid size and harmonics to use. + *

    + * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 + */ public class SphericalHarmonicsHigh extends SphericalHarmonics { - /** - * Constructor for SphericalHarmonics feature module. Uses the values a higher - * grid-size value than proposed in [1] and harmonics up to l=5 - */ - public SphericalHarmonicsHigh() { - super("features_sphericalhhigh", 74, 1, 5); - } + + /** + * Constructor for SphericalHarmonics feature module. Uses the values a higher grid-size value than proposed in [1] and harmonics up to l=5 + */ + public SphericalHarmonicsHigh() { + super("features_sphericalhhigh", 74, 1, 5); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsLow.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsLow.java index 79339e9c2..799f0c5f4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsLow.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SphericalHarmonicsLow.java @@ -1,19 +1,16 @@ package org.vitrivr.cineast.core.features; /** - * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version - * uses low-resolution settings in terms of harmonics to consider. - * - * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). - * A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 - - */ + * An Extraction and Retrieval module for 3D models that leverages Spherical Harmonics as proposed in [1]. This version uses low-resolution settings in terms of harmonics to consider. + *

    + * [1] Funkhouser, T., Min, P., Kazhdan, M., Chen, J., Halderman, A., Dobkin, D., & Jacobs, D. (2003). A search engine for 3D models. ACM Trans. Graph., 22(1), 83–105. http://doi.org/10.1145/588272.588279 + */ public class SphericalHarmonicsLow extends SphericalHarmonics { - /** - * Constructor for SphericalHarmonics feature module. Uses the values - * for grid_size proposed in [1] and harmonics up to l=5 - */ - public SphericalHarmonicsLow() { - super("features_sphericalhlow", 64, 0,3); - } + + /** + * Constructor for SphericalHarmonics feature module. Uses the values for grid_size proposed in [1] and harmonics up to l=5 + */ + public SphericalHarmonicsLow() { + super("features_sphericalhlow", 64, 0, 3); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivAverageFuzzyColor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivAverageFuzzyColor.java index 4853b089f..d394f1bef 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivAverageFuzzyColor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivAverageFuzzyColor.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -13,8 +14,6 @@ import org.vitrivr.cineast.core.extraction.segmenter.SubdividedFuzzyColorHistogram; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class SubDivAverageFuzzyColor extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMedianFuzzyColor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMedianFuzzyColor.java index db4dc6a12..8ea32a3b6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMedianFuzzyColor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMedianFuzzyColor.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.QueryConfig; @@ -13,8 +14,6 @@ import org.vitrivr.cineast.core.extraction.segmenter.SubdividedFuzzyColorHistogram; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; -import java.util.List; - public class SubDivMedianFuzzyColor extends AbstractFeatureModule { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram2.java index 680718c4f..5cdb15599 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram2.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogram2 extends SubDivMotionHistogram { public SubDivMotionHistogram2() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram3.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram3.java index fe2c01b14..7a855a209 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram3.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram3.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogram3 extends SubDivMotionHistogram { public SubDivMotionHistogram3() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram4.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram4.java index 9a061354b..7fb8521c5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram4.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram4.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogram4 extends SubDivMotionHistogram { public SubDivMotionHistogram4() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram5.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram5.java index 0c276b8fa..536d94f67 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram5.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogram5.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogram5 extends SubDivMotionHistogram { public SubDivMotionHistogram5() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground2.java index 7f7675b62..ead12a7fc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground2.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogramBackground2 extends SubDivMotionHistogram { public SubDivMotionHistogramBackground2() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground3.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground3.java index 7ce55a2a1..e83c132c7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground3.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground3.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,45 +12,42 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogramBackground3 extends SubDivMotionHistogram { - public SubDivMotionHistogramBackground3() { - super("features_SubDivMotionHistogramBackground3", "hists", MathHelper.SQRT2 * 9, 3); - } - - @Override - public void processSegment(SegmentContainer shot) { - if(!phandler.idExists(shot.getId())){ - - Pair, ArrayList>> pair = getSubDivHist(3, shot.getBgPaths()); - - FloatVector sum = new FloatVectorImpl(pair.first); - ArrayList tmp = new ArrayList(3 * 3 * 8); - for(List l : pair.second){ - for(float f : l){ - tmp.add(f); - } - } - FloatVectorImpl fv = new FloatVectorImpl(tmp); - - persist(shot.getId(), sum, fv); - } - } - - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - Pair, ArrayList>> pair = getSubDivHist(3, sc.getBgPaths()); - - ArrayList tmp = new ArrayList(3 * 3 * 8); - for (List l : pair.second) { - for (float f : l) { - tmp.add(f); - } - } - FloatVectorImpl fv = new FloatVectorImpl(tmp); + public SubDivMotionHistogramBackground3() { + super("features_SubDivMotionHistogramBackground3", "hists", MathHelper.SQRT2 * 9, 3); + } + + @Override + public void processSegment(SegmentContainer shot) { + if (!phandler.idExists(shot.getId())) { + + Pair, ArrayList>> pair = getSubDivHist(3, shot.getBgPaths()); + + FloatVector sum = new FloatVectorImpl(pair.first); + ArrayList tmp = new ArrayList(3 * 3 * 8); + for (List l : pair.second) { + for (float f : l) { + tmp.add(f); + } + } + FloatVectorImpl fv = new FloatVectorImpl(tmp); + + persist(shot.getId(), sum, fv); + } + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + Pair, ArrayList>> pair = getSubDivHist(3, sc.getBgPaths()); + + ArrayList tmp = new ArrayList(3 * 3 * 8); + for (List l : pair.second) { + for (float f : l) { + tmp.add(f); + } + } + FloatVectorImpl fv = new FloatVectorImpl(tmp); return getSimilar(ReadableFloatVector.toArray(fv), qc); - } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground4.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground4.java index 90be2ea2c..58423ea1d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground4.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground4.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,45 +12,42 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogramBackground4 extends SubDivMotionHistogram { - - public SubDivMotionHistogramBackground4() { - super("features_SubDivMotionHistogramBackground4", "hists", MathHelper.SQRT2 * 16, 4); - } - - @Override - public void processSegment(SegmentContainer shot) { - if(!phandler.idExists(shot.getId())){ - - Pair, ArrayList>> pair = getSubDivHist(4, shot.getBgPaths()); - - FloatVector sum = new FloatVectorImpl(pair.first); - ArrayList tmp = new ArrayList(4 * 4 * 8); - for(List l : pair.second){ - for(float f : l){ - tmp.add(f); - } - } - FloatVectorImpl fv = new FloatVectorImpl(tmp); - - persist(shot.getId(), sum, fv); - } - } - - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - Pair, ArrayList>> pair = getSubDivHist(4, sc.getBgPaths()); - - ArrayList tmp = new ArrayList(4 * 4 * 8); - for(List l : pair.second){ - for(float f : l){ - tmp.add(f); - } - } - FloatVectorImpl fv = new FloatVectorImpl(tmp); + + public SubDivMotionHistogramBackground4() { + super("features_SubDivMotionHistogramBackground4", "hists", MathHelper.SQRT2 * 16, 4); + } + + @Override + public void processSegment(SegmentContainer shot) { + if (!phandler.idExists(shot.getId())) { + + Pair, ArrayList>> pair = getSubDivHist(4, shot.getBgPaths()); + + FloatVector sum = new FloatVectorImpl(pair.first); + ArrayList tmp = new ArrayList(4 * 4 * 8); + for (List l : pair.second) { + for (float f : l) { + tmp.add(f); + } + } + FloatVectorImpl fv = new FloatVectorImpl(tmp); + + persist(shot.getId(), sum, fv); + } + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + Pair, ArrayList>> pair = getSubDivHist(4, sc.getBgPaths()); + + ArrayList tmp = new ArrayList(4 * 4 * 8); + for (List l : pair.second) { + for (float f : l) { + tmp.add(f); + } + } + FloatVectorImpl fv = new FloatVectorImpl(tmp); return getSimilar(ReadableFloatVector.toArray(fv), qc); - } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground5.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground5.java index 25c94e920..ee054293e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground5.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionHistogramBackground5.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; @@ -10,9 +12,6 @@ import org.vitrivr.cineast.core.features.abstracts.SubDivMotionHistogram; import org.vitrivr.cineast.core.util.MathHelper; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionHistogramBackground5 extends SubDivMotionHistogram { public SubDivMotionHistogramBackground5() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum2.java index 4e31d4928..0534929f0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum2.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSum2 extends MotionHistogramCalculator { public SubDivMotionSum2() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum3.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum3.java index 646ad2a36..9310011b2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum3.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum3.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSum3 extends MotionHistogramCalculator { public SubDivMotionSum3() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum4.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum4.java index ae526a2a3..52ff159f6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum4.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum4.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSum4 extends MotionHistogramCalculator { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum5.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum5.java index 3e3b8acbd..83a3b6bcd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum5.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSum5.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSum5 extends MotionHistogramCalculator { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground2.java index f3ed35c19..d9fe71e7d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground2.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSumBackground2 extends MotionHistogramCalculator { public SubDivMotionSumBackground2() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground3.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground3.java index f24fc46e5..371d1785b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground3.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground3.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSumBackground3 extends MotionHistogramCalculator { public SubDivMotionSumBackground3() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground4.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground4.java index 7d1f5be52..dafc12a93 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground4.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground4.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,9 +10,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSumBackground4 extends MotionHistogramCalculator { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground5.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground5.java index 6a1563ec9..37b4f931b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground5.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SubDivMotionSumBackground5.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.features; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.Pair; @@ -8,21 +10,18 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MotionHistogramCalculator; -import java.util.ArrayList; -import java.util.List; - public class SubDivMotionSumBackground5 extends MotionHistogramCalculator { - - public SubDivMotionSumBackground5() { - super("features.SubDivMotionHistogramBackground5", "sums", 2500, 5); - } - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - Pair, ArrayList>> pair = getSubDivHist(5, sc.getBgPaths()); - FloatVectorImpl fv = new FloatVectorImpl(pair.first); + public SubDivMotionSumBackground5() { + super("features.SubDivMotionHistogramBackground5", "sums", 2500, 5); + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + Pair, ArrayList>> pair = getSubDivHist(5, sc.getBgPaths()); + FloatVectorImpl fv = new FloatVectorImpl(pair.first); return getSimilar(ReadableFloatVector.toArray(fv), qc); - } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TemporalDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TemporalDistance.java index c3119c142..f95b90173 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TemporalDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TemporalDistance.java @@ -1,6 +1,9 @@ package org.vitrivr.cineast.core.features; import com.google.common.collect.ImmutableList; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.CorrespondenceFunction; import org.vitrivr.cineast.core.data.GpsData; @@ -9,25 +12,19 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.MetadataFeatureModule; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - // TODO: Change from Instant to Range /** - * A feature that calculates similarity based on the distance between the timestamps of two objects - * measured in seconds. + * A feature that calculates similarity based on the distance between the timestamps of two objects measured in seconds. * *

    It extracts the timestamp provided by Exif GPS data, if available, * or a supplementary JSON file. See {@link GpsData} for more information. * *

    During retrieval, it does nearest neighbor search using the euclidean distance. As of now, - * it uses a hyperbolic correspondence function scaled to days, i.e. 13h20min correspond to - * 90% similarity, 5 days to 50% and 45 days to 10%. This correspondence is likely unfit for many - * datasets and needs to be adjusted accordingly. + * it uses a hyperbolic correspondence function scaled to days, i.e. 13h20min correspond to 90% similarity, 5 days to 50% and 45 days to 10%. This correspondence is likely unfit for many datasets and needs to be adjusted accordingly. */ public class TemporalDistance extends MetadataFeatureModule { + private static final String METADATA_DOMAIN = "TIME"; private static final String FEATURE_NAME = "features_TemporalDistance"; private static final float TIME_SCALE = 5 * 24 * 60 * 60 * 1000; // 5 days, 30 * 60 * 1000; // 30min in ms @@ -65,13 +62,10 @@ public Optional extractFeature(SegmentContainer segmentContainer) } /** - * Extracts the timestamp from the specified path using Exif GPS data. If - * not present, it retrieves the coordinates from a complementary JSON file named after the - * original document. See {@link GpsData} for more information. + * Extracts the timestamp from the specified path using Exif GPS data. If not present, it retrieves the coordinates from a complementary JSON file named after the original document. See {@link GpsData} for more information. * * @param object Path to the file for which the coordinates should be extracted. - * @return an {@link Optional} containing an {@link InstantVector} based on the extracted Exif or - * JSON timestamp, if found, otherwise an empty {@code Optional}. + * @return an {@link Optional} containing an {@link InstantVector} based on the extracted Exif or JSON timestamp, if found, otherwise an empty {@code Optional}. */ @Override public Optional extractFeature(String objectId, Path object) { @@ -80,7 +74,7 @@ public Optional extractFeature(String objectId, Path object) { @Override public List createDescriptors(String objectId, - InstantVector feature) { + InstantVector feature) { return ImmutableList.of(MediaObjectMetadataDescriptor .of(objectId, this.domain(), GpsData.KEY_DATETIME, feature.getInstant().toString())); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/WSDMTICollectionBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/WSDMTICollectionBooleanRetriever.java index 00ae4b2e6..268fdec6c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/WSDMTICollectionBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/WSDMTICollectionBooleanRetriever.java @@ -1,11 +1,11 @@ package org.vitrivr.cineast.core.features; -import org.vitrivr.cineast.core.features.retriever.CollectionBooleanRetriever; - import java.util.Arrays; +import org.vitrivr.cineast.core.features.retriever.CollectionBooleanRetriever; public class WSDMTICollectionBooleanRetriever extends CollectionBooleanRetriever { - public WSDMTICollectionBooleanRetriever() { - super("features_wsdmtiannotations", Arrays.asList("annotation","object")); - } + + public WSDMTICollectionBooleanRetriever() { + super("features_wsdmtiannotations", Arrays.asList("annotation", "object")); + } } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java index 9988bf9d1..39548e4da 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java @@ -6,125 +6,123 @@ import boofcv.struct.feature.BrightFeature; import boofcv.struct.feature.TupleDesc_F64; import boofcv.struct.image.GrayF32; +import java.util.List; import org.ddogleg.clustering.AssignCluster; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.db.DBSelectorSupplier; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import java.util.List; - /** - * An abstract feature module that leverages a named codebook and a set of features to obtain - * a histogram of codewords. It remains to the implementer which codebook and what descriptors to use. - * Once features have been obtained, use the histogram() method to get the histogram given the corpus. - * + * An abstract feature module that leverages a named codebook and a set of features to obtain a histogram of codewords. It remains to the implementer which codebook and what descriptors to use. Once features have been obtained, use the histogram() method to get the histogram given the corpus. + *

    * All codebooks should be placed in the ./resources/codebooks folder. - * + *

    * This class currently requires BoofCV. - * + *

    * TODO: Use CSV based format for codebooks. - * */ public abstract class AbstractCodebookFeatureModule extends StagedFeatureModule { - /** The Assignment used for the codebook. */ - private AssignCluster assignment; - - /** The folder that contains the Codebook(s). */ - private static String CODEBOOK_FOLDER = "resources/codebooks/"; - protected AbstractCodebookFeatureModule(String tableName, float maxDist, int vectorLength) { - super(tableName, maxDist, vectorLength); + /** + * The Assignment used for the codebook. + */ + private AssignCluster assignment; + + /** + * The folder that contains the Codebook(s). + */ + private static String CODEBOOK_FOLDER = "resources/codebooks/"; + + protected AbstractCodebookFeatureModule(String tableName, float maxDist, int vectorLength) { + super(tableName, maxDist, vectorLength); + } + + /** + * Initializer for Extraction - must load the codebook. + */ + @Override + public final void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + super.init(phandlerSupply, batchSize); + + /* Load the Codebook. */ + this.assignment = UtilIO.load(CODEBOOK_FOLDER + this.codebook()); + } + + /** + * Initializer for Retrieval - must load the codebook.selectorSupply + */ + @Override + public final void init(DBSelectorSupplier selectorSupply) { + super.init(selectorSupply); + + /* Load the Codebook. */ + this.assignment = UtilIO.load(CODEBOOK_FOLDER + this.codebook()); + } + + /** + * Returns a histogram given the provided descriptors and the assignment object loaded from the codebook. + * + * @param hard Indicates whether to use hard or soft assignment. + * @param descriptors Feature descriptors. + * @return float[] array with codebook + */ + protected final float[] histogram(boolean hard, DetectDescribePoint descriptors) { + /* Create new Histogram-Calculator. */ + FeatureToWordHistogram_F64 histogram = new FeatureToWordHistogram_F64(this.assignment, hard); + + /* Add the features to the Histogram-Calculator... */ + for (int i = 0; i < descriptors.getNumberOfFeatures(); i++) { + histogram.addFeature(descriptors.getDescription(i)); } - /** - * Initializer for Extraction - must load the codebook. - */ - @Override - public final void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - super.init(phandlerSupply, batchSize); - - /* Load the Codebook. */ - this.assignment = UtilIO.load(CODEBOOK_FOLDER + this.codebook()); - } - - /** - * Initializer for Retrieval - must load the codebook.selectorSupply - */ - @Override - public final void init(DBSelectorSupplier selectorSupply) { - super.init(selectorSupply); - - /* Load the Codebook. */ - this.assignment = UtilIO.load(CODEBOOK_FOLDER + this.codebook()); + /* ... and calculates and returns the histogram. */ + histogram.process(); + return this.floatToDoubleArray(histogram.getHistogram()); + } + + /** + * Returns a histogram given the provided descriptors and the assignment object loaded from the codebook. + * + * @param hard Indicates whether to use hard or soft assignment. + * @param descriptors Feature descriptors as List of TupleDesc_F64 + * @return float[] array with codebook + */ + protected final float[] histogram(boolean hard, List descriptors) { + /* Create new Histogram-Calculator. */ + FeatureToWordHistogram_F64 histogram = new FeatureToWordHistogram_F64(this.assignment, hard); + + /* Add the features to the Histogram-Calculator... */ + for (TupleDesc_F64 descriptor : descriptors) { + histogram.addFeature(descriptor); } - /** - * Returns a histogram given the provided descriptors and the assignment object loaded - * from the codebook. - * - * @param hard Indicates whether to use hard or soft assignment. - * @param descriptors Feature descriptors. - * @return float[] array with codebook - */ - protected final float[] histogram(boolean hard, DetectDescribePoint descriptors) { - /* Create new Histogram-Calculator. */ - FeatureToWordHistogram_F64 histogram = new FeatureToWordHistogram_F64(this.assignment, hard); - - /* Add the features to the Histogram-Calculator... */ - for (int i=0;i descriptors) { - /* Create new Histogram-Calculator. */ - FeatureToWordHistogram_F64 histogram = new FeatureToWordHistogram_F64(this.assignment, hard); - - /* Add the features to the Histogram-Calculator... */ - for (TupleDesc_F64 descriptor : descriptors) { - histogram.addFeature(descriptor); - } - - /* ... and calculates and returns the histogram. */ - histogram.process(); - return this.floatToDoubleArray(histogram.getHistogram()); - } - - /** - * Converts a double array into a float array of the same size. - * - * @param dbl double array to be converted.. - * @return float array - */ - protected final float[] floatToDoubleArray(double[] dbl) { - float[] flt = new float[dbl.length]; - for (int i=0;i phandler; - protected CorrespondenceFunction correspondence; - - protected AbstractFeatureModule(String tableName, float maxDist, int vectorLength) { - this.tableName = tableName; - this.maxDist = maxDist; - this.vectorLength = vectorLength; - this.correspondence = CorrespondenceFunction.linear(maxDist); - } - - @Override - public List getTableNames() { - if (this.tableName == null) { - return new ArrayList<>(); - } - return Collections.singletonList(this.tableName); - } - - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - this.phandler = phandlerSupply.get(); - this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); - this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); - } - - @Override - public void init(DBSelectorSupplier selectorSupply) { - this.selector = selectorSupply.get(); - this.selector.open(this.tableName); - } - - protected void persist(String shotId, ReadableFloatVector fv) { - SimpleFeatureDescriptor descriptor = new SimpleFeatureDescriptor(shotId, fv); - this.writer.write(descriptor); - } - - protected void persist(String shotId, PrimitiveTypeProvider fv) { - SimplePrimitiveTypeProviderFeatureDescriptor descriptor = new SimplePrimitiveTypeProviderFeatureDescriptor(shotId, fv); - this.primitiveWriter.write(descriptor); - } - - protected void persist(String shotId, List fvs) { - List entities = fvs.stream().map(fv -> new SimpleFeatureDescriptor(shotId, fv)).collect(Collectors.toList()); - this.writer.write(entities); - } - - protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { - return new QueryConfig(qc).setCorrespondenceFunctionIfEmpty(this.correspondence); - } - - @Override - public List getSimilar(String segmentId, ReadableQueryConfig qc) { - List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER); - if (list.isEmpty()) { - LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); - return new ArrayList<>(0); - } - return getSimilarHelper(list, qc); - } - - public List getSimilar(List segmentIds, ReadableQueryConfig qc) { - List list = this.selector.getRows(GENERIC_ID_COLUMN_QUALIFIER, segmentIds).stream().map(map -> map.get(FEATURE_COLUMN_QUALIFIER)).collect(Collectors.toList()); - - if (list.isEmpty()) { - LOGGER.warn("No feature vectors for segmentIds {} found, returning empty result-list", segmentIds); - return new ArrayList<>(0); - } - return getSimilarHelper(list, qc); - } - - protected List getSimilarHelper(List list, ReadableQueryConfig qc) { - if (list.size() == 1) { - return getSimilar(list.get(0), qc); - } - - List vectors = list.stream().map(FloatArrayProvider::getFloatArray).collect(Collectors.toList()); - List distances = this.selector.getBatchedNearestNeighbours(qc.getResultsPerModule(), vectors, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, vectors.stream().map(x -> setQueryConfig(qc)).collect(Collectors.toList())); - CorrespondenceFunction function = qc.getCorrespondenceFunction().orElse(correspondence); - return ScoreElement.filterMaximumScores(DistanceElement.toScore(distances, function).stream()); - } - - /** - * helper function to retrieve elements close to a vector which has to be generated by the feature module. - */ - protected List getSimilar(float[] vector, ReadableQueryConfig qc) { - return getSimilar(new FloatArrayTypeProvider(vector), qc); - } - - /** - * Helper function to retrieve elements close to a generic primitive type - */ - protected List getSimilar(PrimitiveTypeProvider queryProvider, ReadableQueryConfig qc) { - ReadableQueryConfig qcc = setQueryConfig(qc); - List distances = this.selector.getNearestNeighboursGeneric(qc.getResultsPerModule(), queryProvider, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, qcc); - CorrespondenceFunction function = qcc.getCorrespondenceFunction().orElse(correspondence); - return DistanceElement.toScore(distances, function); - } - - @Override - public void finish() { - if (this.writer != null) { - this.writer.close(); - this.writer = null; - } - - if (this.primitiveWriter != null) { - this.primitiveWriter.close(); - this.primitiveWriter = null; - } - - if (this.phandler != null) { - this.phandler.close(); - this.phandler = null; - } - - if (this.selector != null) { - this.selector.close(); - this.selector = null; - } - } - - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity(this.tableName, true, this.vectorLength); - } - - @Override - public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity(this.tableName); - } + private static final Logger LOGGER = LogManager.getLogger(); + protected SimpleFeatureDescriptorWriter writer; + protected PrimitiveTypeProviderFeatureDescriptorWriter primitiveWriter; + protected DBSelector selector; + protected final float maxDist; + protected final int vectorLength; + protected final String tableName; + protected PersistencyWriter phandler; + protected CorrespondenceFunction correspondence; + + protected AbstractFeatureModule(String tableName, float maxDist, int vectorLength) { + this.tableName = tableName; + this.maxDist = maxDist; + this.vectorLength = vectorLength; + this.correspondence = CorrespondenceFunction.linear(maxDist); + } + + @Override + public List getTableNames() { + if (this.tableName == null) { + return new ArrayList<>(); + } + return Collections.singletonList(this.tableName); + } + + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + this.phandler = phandlerSupply.get(); + this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); + this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); + } + + @Override + public void init(DBSelectorSupplier selectorSupply) { + this.selector = selectorSupply.get(); + this.selector.open(this.tableName); + } + + protected void persist(String shotId, ReadableFloatVector fv) { + SimpleFeatureDescriptor descriptor = new SimpleFeatureDescriptor(shotId, fv); + this.writer.write(descriptor); + } + + protected void persist(String shotId, PrimitiveTypeProvider fv) { + SimplePrimitiveTypeProviderFeatureDescriptor descriptor = new SimplePrimitiveTypeProviderFeatureDescriptor(shotId, fv); + this.primitiveWriter.write(descriptor); + } + + protected void persist(String shotId, List fvs) { + List entities = fvs.stream().map(fv -> new SimpleFeatureDescriptor(shotId, fv)).collect(Collectors.toList()); + this.writer.write(entities); + } + + protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc).setCorrespondenceFunctionIfEmpty(this.correspondence); + } + + @Override + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER); + if (list.isEmpty()) { + LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); + return new ArrayList<>(0); + } + return getSimilarHelper(list, qc); + } + + public List getSimilar(List segmentIds, ReadableQueryConfig qc) { + List list = this.selector.getRows(GENERIC_ID_COLUMN_QUALIFIER, segmentIds).stream().map(map -> map.get(FEATURE_COLUMN_QUALIFIER)).collect(Collectors.toList()); + + if (list.isEmpty()) { + LOGGER.warn("No feature vectors for segmentIds {} found, returning empty result-list", segmentIds); + return new ArrayList<>(0); + } + return getSimilarHelper(list, qc); + } + + protected List getSimilarHelper(List list, ReadableQueryConfig qc) { + if (list.size() == 1) { + return getSimilar(list.get(0), qc); + } + + List vectors = list.stream().map(FloatArrayProvider::getFloatArray).collect(Collectors.toList()); + List distances = this.selector.getBatchedNearestNeighbours(qc.getResultsPerModule(), vectors, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, vectors.stream().map(x -> setQueryConfig(qc)).collect(Collectors.toList())); + CorrespondenceFunction function = qc.getCorrespondenceFunction().orElse(correspondence); + return ScoreElement.filterMaximumScores(DistanceElement.toScore(distances, function).stream()); + } + + /** + * helper function to retrieve elements close to a vector which has to be generated by the feature module. + */ + protected List getSimilar(float[] vector, ReadableQueryConfig qc) { + return getSimilar(new FloatArrayTypeProvider(vector), qc); + } + + /** + * Helper function to retrieve elements close to a generic primitive type + */ + protected List getSimilar(PrimitiveTypeProvider queryProvider, ReadableQueryConfig qc) { + ReadableQueryConfig qcc = setQueryConfig(qc); + List distances = this.selector.getNearestNeighboursGeneric(qc.getResultsPerModule(), queryProvider, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, qcc); + CorrespondenceFunction function = qcc.getCorrespondenceFunction().orElse(correspondence); + return DistanceElement.toScore(distances, function); + } + + @Override + public void finish() { + if (this.writer != null) { + this.writer.close(); + this.writer = null; + } + + if (this.primitiveWriter != null) { + this.primitiveWriter.close(); + this.primitiveWriter = null; + } + + if (this.phandler != null) { + this.phandler.close(); + this.phandler = null; + } + + if (this.selector != null) { + this.selector.close(); + this.selector = null; + } + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(this.tableName, true, this.vectorLength); + } + + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity(this.tableName); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index 86e8eea71..d91cefb7f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -87,7 +87,7 @@ public void processSegment(SegmentContainer shot) { /** * Initializes the persistent layer with two fields: GENERIC_ID_COLUMN_QUALIFIER and FEATURE_COLUMN_QUALIFIER both using the Apache Solr storage handler. - * + *

    * This corresponds to the Fieldnames of the {@link SimpleFulltextFeatureDescriptor} The FEATURE_COLUMN_QUALIFIER in this context is the full text for the given segment */ @Override @@ -179,7 +179,7 @@ protected String enrichQueryTerm(String queryTerm) { /** * Convenience-Method for implementing classes once they have generated their query terms. - * + *

    * If there are multiple scores per segment (e.g. a segment has "hello" and "hello world" which produces two hits, does maxpooling */ protected List getSimilar(ReadableQueryConfig qc, String... terms) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java index df5f0798a..2d4dd8aff 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java @@ -2,6 +2,16 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,15 +27,9 @@ import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.retriever.MultipleInstantiatableRetriever; -import org.vitrivr.cineast.core.features.retriever.Retriever; - -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; /** - * BooleanRetrievers operate on tables which are created externally. - * Therefore, both initializing and dropping the entity are managed externally. + * BooleanRetrievers operate on tables which are created externally. Therefore, both initializing and dropping the entity are managed externally. */ public abstract class BooleanRetriever implements MultipleInstantiatableRetriever { @@ -35,18 +39,18 @@ public abstract class BooleanRetriever implements MultipleInstantiatableRetrieve protected final HashSet attributes = new HashSet<>(); protected final HashMap columnTypes = new HashMap<>(); - protected BooleanRetriever(String entity, Collection attributes){ + protected BooleanRetriever(String entity, Collection attributes) { this.entity = entity; this.attributes.addAll(attributes); } - protected BooleanRetriever(Map properties){ - if(!properties.containsKey("entity")){ + protected BooleanRetriever(Map properties) { + if (!properties.containsKey("entity")) { throw new RuntimeException("no entity specified in properties map of BooleanRetriever"); } this.entity = properties.get("entity"); - if(properties.containsKey("attribute")){ + if (properties.containsKey("attribute")) { List attrs = Arrays.stream(properties.get("attribute").split(",")).map(String::trim) .collect( Collectors.toList()); @@ -68,11 +72,11 @@ public void init(DBSelectorSupplier selectorSupply) { this.selector.open(entity); } - public Collection getAttributes(){ + public Collection getAttributes() { return this.attributes.stream().map(x -> this.entity + "." + x).collect(Collectors.toSet()); } - protected boolean canProcess(BooleanExpression be){ + protected boolean canProcess(BooleanExpression be) { return getSupportedOperators().contains(be.getOperator()) && getAttributes().contains(be.getAttribute()); } @@ -81,7 +85,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc List relevantExpressions = sc.getBooleanExpressions().stream().filter(this::canProcess).collect(Collectors.toList()); - if (relevantExpressions.isEmpty()){ + if (relevantExpressions.isEmpty()) { LOGGER.debug("No relevant expressions in {} for query {}", this.getClass().getSimpleName(), sc.toString()); return Collections.emptyList(); } @@ -89,12 +93,12 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc return getMatching(relevantExpressions, qc); } - protected List getMatching(List expressions, ReadableQueryConfig qc){ + protected List getMatching(List expressions, ReadableQueryConfig qc) { List> rows = selector.getRowsAND( expressions.stream().map(be -> Triple.of( // strip entity if it was given via config - be.getAttribute().contains(this.entity) ? be.getAttribute().substring(this.entity.length()+1) : be.getAttribute(), + be.getAttribute().contains(this.entity) ? be.getAttribute().substring(this.entity.length() + 1) : be.getAttribute(), be.getOperator(), be.getValues() )).collect(Collectors.toList()), @@ -125,7 +129,7 @@ public void dropPersistentLayer(Supplier supply) { //nop } - public ProviderDataType getColumnType(String column){ + public ProviderDataType getColumnType(String column) { return this.columnTypes.get(column); } @@ -137,10 +141,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if(obj == null){ + if (obj == null) { return false; } - if(!(obj instanceof BooleanRetriever)){ + if (!(obj instanceof BooleanRetriever)) { return false; } return this.hashCode() == obj.hashCode(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java index aadbe2a95..f32a80058 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java @@ -4,10 +4,13 @@ import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import boofcv.abst.scene.ImageClassifier.Score; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Optional; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; @@ -29,23 +32,17 @@ import org.vitrivr.cineast.core.db.dao.reader.MediaSegmentReader; import org.vitrivr.cineast.core.db.dao.writer.SimpleFeatureDescriptorWriter; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.core.extraction.metadata.MetadataFeatureExtractor; - -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.function.Supplier; +import org.vitrivr.cineast.core.features.retriever.Retriever; /** - * Feature module that bases its feature data on the object (usually metadata) itself instead of - * individual segments by combining {@link MetadataFeatureExtractor} and {@link Retriever}. + * Feature module that bases its feature data on the object (usually metadata) itself instead of individual segments by combining {@link MetadataFeatureExtractor} and {@link Retriever}. * * @param the specific type of the feature data */ public abstract class MetadataFeatureModule implements MetadataFeatureExtractor, Retriever { + private static final String ID_COLUMN_NAME = GENERIC_ID_COLUMN_QUALIFIER; private static final String FEATURE_COLUMN_NAME = FEATURE_COLUMN_QUALIFIER; @@ -65,13 +62,19 @@ protected MetadataFeatureModule(int vectorLength, Map properties segmentRetrievalScope = Boolean.parseBoolean(properties.getOrDefault("segmentRetrievalScope", "false")); } - /** Returns the name of the feature entity as stored in the persistent layer. */ + /** + * Returns the name of the feature entity as stored in the persistent layer. + */ public abstract String featureEntityName(); - /** Returns the default distance if none is set. */ + /** + * Returns the default distance if none is set. + */ public abstract Distance defaultDistance(); - /** Returns the default correspondence function if none is set. */ + /** + * Returns the default correspondence function if none is set. + */ public abstract CorrespondenceFunction defaultCorrespondence(); @Override @@ -80,8 +83,7 @@ public List getTableNames() { } /** - * Returns an {@link Optional} containing the extracted feature data from the segment container, - * if available, otherwise an empty {@code Optional}. + * Returns an {@link Optional} containing the extracted feature data from the segment container, if available, otherwise an empty {@code Optional}. */ public abstract Optional extractFeature(SegmentContainer segmentContainer); @@ -180,10 +182,10 @@ protected List getSimilar(float[] feature, ReadableQueryConfig rqc List oDistances = new ArrayList<>(); CorrespondenceFunction correspondence = qc.getCorrespondenceFunction() .orElse(this.defaultCorrespondence()); - if (this.segmentRetrievalScope){ + if (this.segmentRetrievalScope) { sDistances = this.dbSelector.getNearestNeighboursGeneric(rqc.getResultsPerModule(), feature, FEATURE_COLUMN_NAME, SegmentDistanceElement.class, qc); return DistanceElement.toScore(sDistances, correspondence); - }else{ + } else { oDistances = this.dbSelector.getNearestNeighboursGeneric(rqc.getResultsPerModule(), feature, FEATURE_COLUMN_NAME, ObjectDistanceElement.class, qc); return DistanceElement.toScore(oDistances, correspondence); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java index be4acf10e..db52b3d4d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MotionHistogramCalculator.java @@ -3,9 +3,12 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import georegression.struct.point.Point2D_F32; - -import java.util.*; - +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; @@ -22,8 +25,6 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.retriever.Retriever; -import java.util.function.Supplier; - public abstract class MotionHistogramCalculator implements Retriever { protected DBSelector selector; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java index 87702e9c3..ec6abd7e4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java @@ -14,182 +14,167 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; /** - * This implementation of the AbstractFeatureModule executes every query, either based on a SegmentContainer - * or on an existing segment, in three stages. This for in-depth analysis and benchmarking of the different - * stages as well as a unified approach to similarity search. - * - * When implementing this class, you are expected to override the methods that represent the different stages - * instead of implementing the getSimilar() methods. - * + * This implementation of the AbstractFeatureModule executes every query, either based on a SegmentContainer or on an existing segment, in three stages. This for in-depth analysis and benchmarking of the different stages as well as a unified approach to similarity search. + *

    + * When implementing this class, you are expected to override the methods that represent the different stages instead of implementing the getSimilar() methods. */ public abstract class StagedFeatureModule extends AbstractFeatureModule { - protected static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); - - /** - * Constructor - * - * @param tableName Name of the entity / table to persist data with and read data from. - * @param maxDist Maximum distance value (for normalization). - * @param vectorLength Dimensionality of the feature vector. - */ - protected StagedFeatureModule(String tableName, float maxDist, int vectorLength) { - super(tableName, maxDist, vectorLength); + protected static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); + + /** + * Constructor + * + * @param tableName Name of the entity / table to persist data with and read data from. + * @param maxDist Maximum distance value (for normalization). + * @param vectorLength Dimensionality of the feature vector. + */ + protected StagedFeatureModule(String tableName, float maxDist, int vectorLength) { + super(tableName, maxDist, vectorLength); + } + + /** + * This method executes a regular similarity query based on a provided SegmentContainer. The query is executed in three stages (hence the name of the class): + * + *

      + *
    1. Pre-processing: Extracting features from the SegmentContainer.
      2. + *
    2. Similarity search: Performing the similarity query in the underlying storage engine.
      3. + *
    3. Post-processing: Aggregating the query results into the final Score elements.
      4. + *
    + *

    + * Even though it is possible to re-implement this method, it is not recommended. Instead, try to override + * the methods that represent the different stages. + * + * @param sc SegmentContainer to base the query on. + * @param qc QueryConfiguration + * @return List of results + */ + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + /* Load default query-config. */ + QueryConfig qcc = this.defaultQueryConfig(qc); + + /* Extract features. */ + List features = this.preprocessQuery(sc, qcc); + + if (features == null || features.isEmpty()) { + LOGGER.warn("No features could be generated from the provided query. Aborting query execution..."); + return new ArrayList<>(0); } - /** - * This method executes a regular similarity query based on a provided SegmentContainer. The query - * is executed in three stages (hence the name of the class): - * - *

      - *
    1. Pre-processing: Extracting features from the SegmentContainer.
      2. - *
    2. Similarity search: Performing the similarity query in the underlying storage engine.
      3. - *
    3. Post-processing: Aggregating the query results into the final Score elements.
      4. - *
    - * - * Even though it is possible to re-implement this method, it is not recommended. Instead, try to override - * the methods that represent the different stages. - * - * @param sc SegmentContainer to base the query on. - * @param qc QueryConfiguration - * @return List of results - */ - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - /* Load default query-config. */ - QueryConfig qcc = this.defaultQueryConfig(qc); - - /* Extract features. */ - List features = this.preprocessQuery(sc, qcc); - - if (features == null || features.isEmpty()) { - LOGGER.warn("No features could be generated from the provided query. Aborting query execution..."); - return new ArrayList<>(0); - } - - /* Generates a list of QueryConfigs for the feature. */ - List configs = this.generateQueryConfigsForFeatures(qcc, features); - - /* Start query lookup phase. */ - List partialResults = this.lookup(features, configs); - - /* Start query-results post-processing phase. */ - - return this.postprocessQuery(partialResults, qcc); + /* Generates a list of QueryConfigs for the feature. */ + List configs = this.generateQueryConfigsForFeatures(qcc, features); + + /* Start query lookup phase. */ + List partialResults = this.lookup(features, configs); + + /* Start query-results post-processing phase. */ + + return this.postprocessQuery(partialResults, qcc); + } + + /** + * This method executes a similarity query based on an existing segment. The query is executed in three stages (hence the name of the class): + * + *
      + *
    1. Lookup: Retrieving the features associated with the provided segment ID.
      2. + *
    2. Similarity search: Performing the similarity query in the underlying storage engine.
      3. + *
    3. Post-processing: Aggregating the query results into the final Score elements.
      4. + *
    + *

    + * Even though it is possible to re-implement this method, it is not recommended. Instead, try to override + * the methods that represent the different stages. + * + * @param segmentId ID of the segment that is used as example. + * @param qc QueryConfiguration + * @return List of results + */ + @Override + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + /* Load default query-config. */ + QueryConfig qcc = this.defaultQueryConfig(qc); + + /* Lookup features. */ + List features = this.selector.getFeatureVectors(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER); + if (features.isEmpty()) { + LOGGER.warn("No features could be fetched for the provided segmentId '{}'. Aborting query execution...", segmentId); + return new ArrayList<>(0); } - /** - * This method executes a similarity query based on an existing segment. The query is executed in three stages - * (hence the name of the class): - * - *

      - *
    1. Lookup: Retrieving the features associated with the provided segment ID.
      2. - *
    2. Similarity search: Performing the similarity query in the underlying storage engine.
      3. - *
    3. Post-processing: Aggregating the query results into the final Score elements.
      4. - *
    - * - * Even though it is possible to re-implement this method, it is not recommended. Instead, try to override - * the methods that represent the different stages. - * - * @param segmentId ID of the segment that is used as example. - * @param qc QueryConfiguration - * - * @return List of results - */ - @Override - public List getSimilar(String segmentId, ReadableQueryConfig qc) { - /* Load default query-config. */ - QueryConfig qcc = this.defaultQueryConfig(qc); - - /* Lookup features. */ - List features = this.selector.getFeatureVectors(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER); - if (features.isEmpty()) { - LOGGER.warn("No features could be fetched for the provided segmentId '{}'. Aborting query execution...", segmentId); - return new ArrayList<>(0); - } - - /* Generate a list of QueryConfigs for the feature. */ - List configs = this.generateQueryConfigsForFeatures(qcc, features); - - /* Start query lookup phase. */ - List partialResults = this.lookup(features, configs); - - /* Start query-results post-processing phase. */ - return this.postprocessQuery(partialResults, qcc); + /* Generate a list of QueryConfigs for the feature. */ + List configs = this.generateQueryConfigsForFeatures(qcc, features); + + /* Start query lookup phase. */ + List partialResults = this.lookup(features, configs); + + /* Start query-results post-processing phase. */ + return this.postprocessQuery(partialResults, qcc); + } + + /** + * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector. + * + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param sc SegmentContainer that was submitted to the feature module + * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. + * @return A pair containing a List of features and an optional weight vector. + */ + protected abstract List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc); + + /** + * This method represents the lookup step that's executed when processing a query. Lookup is based on the feature vectors returned by the first stage and a lookup is executed for each. Partial-results are accumulated in a list, which is returned by the method at the end. + * + * Important: The weight-vector must have the same size as the feature-vectors returned by the method. + * + * @param features A list of feature-vectors (usually generated in the first stage). For each feature, a lookup is executed. May be empty! + * @param configs A ReadableQueryConfig object that contains query-related configuration parameters. + * @return Unfiltered list of partial results. May exceed the number of results a module is supposed to return and entries may occur multiple times. + */ + protected List lookup(List features, List configs) { + List partialResults; + if (features.size() == 1) { + partialResults = this.selector.getNearestNeighboursGeneric(configs.get(0).getResultsPerModule(), features.get(0), FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs.get(0)); + } else if (features.size() > 1) { + partialResults = this.selector.getBatchedNearestNeighbours(configs.get(0).getResultsPerModule(), features, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs); + } else { + partialResults = new ArrayList<>(0); } - - /** - * This method represents the first step that's executed when processing query. The associated SegmentContainer is - * examined and feature-vectors are being generated. The generated vectors are returned by this method together with an - * optional weight-vector. - * - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param sc SegmentContainer that was submitted to the feature module - * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited. - * @return A pair containing a List of features and an optional weight vector. - */ - protected abstract List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc); - - /** - * This method represents the lookup step that's executed when processing a query. Lookup is based on the feature vectors - * returned by the first stage and a lookup is executed for each. Partial-results are accumulated in a list, which is returned - * by the method at the end. - * - * Important: The weight-vector must have the same size as the feature-vectors returned by the method. - * - * @param features A list of feature-vectors (usually generated in the first stage). For each feature, a lookup is executed. May be empty! - * @param configs A ReadableQueryConfig object that contains query-related configuration parameters. - * @return Unfiltered list of partial results. May exceed the number of results a module is supposed to return and entries may occur multiple times. - */ - protected List lookup(List features, List configs) { - List partialResults; - if (features.size() == 1) { - partialResults = this.selector.getNearestNeighboursGeneric(configs.get(0).getResultsPerModule(), features.get(0), FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs.get(0)); - } else if (features.size() > 1) { - partialResults = this.selector.getBatchedNearestNeighbours(configs.get(0).getResultsPerModule(), features, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, configs); - } else { - partialResults = new ArrayList<>(0); - } - return partialResults; - } - - /** - * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by - * the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of - * ScoreElements is returned by the feature module during retrieval. - * - * @param partialResults List of partial results returned by the lookup stage. - * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module. - */ - protected abstract List postprocessQuery(List partialResults, ReadableQueryConfig qcc); - - /** - * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the - * same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. - * - * @param qc Original query config - * @param features List of features for which a QueryConfig is required. - * @return New query config (may be identical to the original one). - */ - protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { - List configs = new ArrayList<>(features.size()); - for (int i=0;i postprocessQuery(List partialResults, ReadableQueryConfig qcc); + + /** + * Returns a list of QueryConfigs for the given list of features. By default, this method simply returns a list of the same the provided config. However, this method can be re-implemented to e.g. add a static or dynamic weight vectors. + * + * @param qc Original query config + * @param features List of features for which a QueryConfig is required. + * @return New query config (may be identical to the original one). + */ + protected List generateQueryConfigsForFeatures(ReadableQueryConfig qc, List features) { + List configs = new ArrayList<>(features.size()); + for (int i = 0; i < features.size(); i++) { + configs.add(qc); } + return configs; + } + + /** + * Merges the provided QueryConfig with the default QueryConfig enforced by the feature module. + * + * @param qc QueryConfig provided by the caller of the feature module. + * @return Modified QueryConfig. + */ + protected QueryConfig defaultQueryConfig(ReadableQueryConfig qc) { + return new QueryConfig(qc) + .setCorrespondenceFunctionIfEmpty(this.correspondence) + .setDistanceIfEmpty(QueryConfig.Distance.euclidean); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java index e896e81fa..8901a5419 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java @@ -2,6 +2,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.distance.DistanceElement; @@ -12,11 +13,10 @@ import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.features.extractor.Extractor; -import java.util.List; - public abstract class SubDivMotionHistogram extends MotionHistogramCalculator implements Extractor { + protected PersistencyWriter phandler; - + protected SubDivMotionHistogram(String tableName, String fieldName, double maxDist, int cells) { super(tableName, fieldName, (float) maxDist, cells); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/CodebookGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/CodebookGenerator.java index 00fc0896b..f3b8f0fcf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/CodebookGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/CodebookGenerator.java @@ -5,14 +5,14 @@ public interface CodebookGenerator { - /** - * Generates a Codebook for files in the source folder and writes it to the destination folder. - * - * @param source Path pointing to a folder with file from which the codebook should be created. - * @param destination Output file - * @param words Number of words in the Codebook. - * - * @throws IOException If an error occurs while reading the files. - */ - void generate(Path source, Path destination, int words) throws IOException; + + /** + * Generates a Codebook for files in the source folder and writes it to the destination folder. + * + * @param source Path pointing to a folder with file from which the codebook should be created. + * @param destination Output file + * @param words Number of words in the Codebook. + * @throws IOException If an error occurs while reading the files. + */ + void generate(Path source, Path destination, int words) throws IOException; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/HOGCodebookGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/HOGCodebookGenerator.java index 805a07b71..e5d5f71e2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/HOGCodebookGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/HOGCodebookGenerator.java @@ -3,39 +3,38 @@ import boofcv.abst.feature.dense.DescribeImageDense; import boofcv.struct.feature.TupleDesc_F64; import boofcv.struct.image.GrayU8; +import java.awt.image.BufferedImage; import org.ddogleg.clustering.FactoryClustering; import org.vitrivr.cineast.core.util.images.HOGHelper; -import java.awt.image.BufferedImage; - public class HOGCodebookGenerator extends ImageCodebookGenerator { - /** - * Default constructor. - */ - public HOGCodebookGenerator() { - super(HOGHelper.hogVectorSize(HOGHelper.DEFAULT_CONFIG), true); - } - /** - * Initializes the codebook generator (i.e. setup the clusterer etc.) - */ - @Override - protected void init() { - this.clusterer = FactoryClustering.kMeans_F64(null, 200, 20, 1e-7); - } + /** + * Default constructor. + */ + public HOGCodebookGenerator() { + super(HOGHelper.hogVectorSize(HOGHelper.DEFAULT_CONFIG), true); + } + + /** + * Initializes the codebook generator (i.e. setup the clusterer etc.) + */ + @Override + protected void init() { + this.clusterer = FactoryClustering.kMeans_F64(null, 200, 20, 1e-7); + } - /** - * Processes the content (i.e. creates descriptors) and add the generated - * descriptors to the cluster. - * - * @param content The image to process. - */ - @Override - protected void process(BufferedImage content) { - DescribeImageDense hog = HOGHelper.getHOGDescriptors(content); - for (TupleDesc_F64 desc : hog.getDescriptions()) { - this.cluster.addReference(desc); - } + /** + * Processes the content (i.e. creates descriptors) and add the generated descriptors to the cluster. + * + * @param content The image to process. + */ + @Override + protected void process(BufferedImage content) { + DescribeImageDense hog = HOGHelper.getHOGDescriptors(content); + for (TupleDesc_F64 desc : hog.getDescriptions()) { + this.cluster.addReference(desc); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java index 1ba899b32..097395e0a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/codebook/ImageCodebookGenerator.java @@ -2,138 +2,138 @@ import boofcv.alg.bow.ClusterVisualWords; import boofcv.io.UtilIO; -import org.ddogleg.clustering.ComputeClusters; -import org.vitrivr.cineast.core.extraction.decode.general.Decoder; -import org.vitrivr.cineast.core.extraction.decode.image.DefaultImageDecoder; - -import javax.activation.MimetypesFileTypeMap; import java.awt.image.BufferedImage; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayDeque; -import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.activation.MimetypesFileTypeMap; +import org.ddogleg.clustering.ComputeClusters; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.extraction.decode.image.DefaultImageDecoder; /** - * Default implementation of a Codebook generator for images. Extend and add the details like - * the images to use. - * + * Default implementation of a Codebook generator for images. Extend and add the details like the images to use. */ public abstract class ImageCodebookGenerator implements CodebookGenerator { - /** K-Means clusterer used for the clustering step in the Codebook generation. */ - protected ComputeClusters clusterer; - /** The visual words cluster. */ - protected ClusterVisualWords cluster; - - /** - * Default constructor. - * - * @param vectorsize Size of the input vectors (that are getting clustered). - * @param verbose true if the clusterer should print output about its progress. + /** + * K-Means clusterer used for the clustering step in the Codebook generation. + */ + protected ComputeClusters clusterer; + + /** + * The visual words cluster. + */ + protected ClusterVisualWords cluster; + + /** + * Default constructor. + * + * @param vectorsize Size of the input vectors (that are getting clustered). + * @param verbose true if the clusterer should print output about its progress. + */ + public ImageCodebookGenerator(int vectorsize, boolean verbose) { + this.init(); + this.clusterer.setVerbose(verbose); + this.cluster = new ClusterVisualWords(clusterer, vectorsize, 0xA1CF3B12); + } + + @Override + public void generate(Path source, Path destination, int words) throws IOException { + long start = System.currentTimeMillis(); + final Decoder decoder = new DefaultImageDecoder(); + final MimetypesFileTypeMap filetypemap = new MimetypesFileTypeMap("mime.types"); + + /* Filter the list of files and aggregate it. */ + + + /* Prepare array dequeue. */ + ArrayDeque files = Files.walk(source).filter(path -> { + if (decoder.supportedFiles() != null) { + String type = filetypemap.getContentType(path.toString()); + return decoder.supportedFiles().contains(type); + } else { + return true; + } + }).collect(Collectors.toCollection(ArrayDeque::new)); + + /* Prepare data-structures to track progress. */ + int max = files.size(); + int counter = 0; + int skipped = 0; + char[] progressBar = new char[15]; + int update = max / progressBar.length; + + /* */ + System.out.println(String.format("Creating codebook of %d words from %d files.", words, files.size())); + + /* + * Iterates over the files Dequeue. Every element that has been processed in removed from that Dequeue. */ - public ImageCodebookGenerator(int vectorsize, boolean verbose) { - this.init(); - this.clusterer.setVerbose(verbose); - this.cluster = new ClusterVisualWords(clusterer, vectorsize,0xA1CF3B12); - } - - @Override - public void generate(Path source, Path destination, int words) throws IOException { - long start = System.currentTimeMillis(); - final Decoder decoder = new DefaultImageDecoder(); - final MimetypesFileTypeMap filetypemap = new MimetypesFileTypeMap("mime.types"); - - /* Filter the list of files and aggregate it. */ - - - /* Prepare array dequeue. */ - ArrayDeque files = Files.walk(source).filter(path -> { - if (decoder.supportedFiles() != null) { - String type = filetypemap.getContentType(path.toString()); - return decoder.supportedFiles().contains(type); - } else { - return true; - } - }).collect(Collectors.toCollection(ArrayDeque::new)); - - /* Prepare data-structures to track progress. */ - int max = files.size(); - int counter = 0; - int skipped = 0; - char[] progressBar = new char[15]; - int update = max/progressBar.length; - - /* */ - System.out.println(String.format("Creating codebook of %d words from %d files.", words, files.size())); - - /* - * Iterates over the files Dequeue. Every element that has been processed in removed from that Dequeue. - */ - Path path = null; - while ((path = files.poll()) != null) { - if (decoder.init(path, null, null)) { - BufferedImage image = decoder.getNext(); - if (image != null) { - this.process(image); - } else { - skipped++; - } - } else { - skipped++; - } - if (counter % update == 0) { - this.updateProgressBar(progressBar, max, counter); - } - System.out.print(String.format("\rAdding vectors to codebook: %d/%d files processed (%d skipped) |%s| (Memory left: %.2f/%.2f GB)", counter,max,skipped, String.valueOf(progressBar), Runtime.getRuntime().freeMemory()/1000000.0f, Runtime.getRuntime().totalMemory()/1000000.0f)); - counter++; + Path path = null; + while ((path = files.poll()) != null) { + if (decoder.init(path, null, null)) { + BufferedImage image = decoder.getNext(); + if (image != null) { + this.process(image); + } else { + skipped++; } - - /* Dispose of unnecessary elements. */ - files = null; - progressBar = null; - - /* Start clustering.*/ - System.out.println(String.format("\nClustering... this could take a while.")); - this.cluster.process(words); - - /* Save file...*/ - System.out.println(String.format("Saving vocabulary with %d entries.", words)); - UtilIO.save(this.cluster.getAssignment(), destination.toString()); - - long duration = System.currentTimeMillis()-start; - System.out.println(String.format("Done! Took me %dhours %dmin %dsec", TimeUnit.MILLISECONDS.toHours(duration), TimeUnit.MILLISECONDS.toMinutes(duration), TimeUnit.MILLISECONDS.toSeconds(duration))); - + } else { + skipped++; + } + if (counter % update == 0) { + this.updateProgressBar(progressBar, max, counter); + } + System.out.print(String.format("\rAdding vectors to codebook: %d/%d files processed (%d skipped) |%s| (Memory left: %.2f/%.2f GB)", counter, max, skipped, String.valueOf(progressBar), Runtime.getRuntime().freeMemory() / 1000000.0f, Runtime.getRuntime().totalMemory() / 1000000.0f)); + counter++; } - /** - * Updates the char-array of the progress-bar. - */ - private void updateProgressBar(char[] progressBar, int max, int counter) { - int progress = (int)(((float)counter/(float)max)*progressBar.length - 1); - for (int i=0;i surf = SURFHelper.getFastSurf(content); - for (int i=0;i surf = SURFHelper.getFastSurf(content); + for (int i = 0; i < surf.getNumberOfFeatures(); i++) { + this.cluster.addReference(surf.getDescription(i)); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java index 5f5d6fdf7..24d76627c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java @@ -1,13 +1,7 @@ package org.vitrivr.cineast.core.features.exporter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.frames.AudioFrame; -import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.extractor.Extractor; -import org.vitrivr.cineast.core.util.LogHelper; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import java.io.IOException; import java.io.OutputStream; @@ -19,120 +13,126 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.function.Supplier; - -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.frames.AudioFrame; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.extractor.Extractor; +import org.vitrivr.cineast.core.util.LogHelper; /** * Exports the audio in a given segment as mono WAV file. - * */ public class AudioSegmentExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - private static final String PROPERTY_NAME_DESTINATION = "destination"; - - /** Destination path for the audio-segment. */ - private Path destination; - - /** - * Default constructor - */ - public AudioSegmentExporter() { - this(new HashMap<>()); - } - - /** - * Default constructor. The AudioSegmentExport can be configured via named properties in the provided HashMap. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where files should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public AudioSegmentExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + private static final Logger LOGGER = LogManager.getLogger(); + + private static final String PROPERTY_NAME_DESTINATION = "destination"; + + /** + * Destination path for the audio-segment. + */ + private Path destination; + + /** + * Default constructor + */ + public AudioSegmentExporter() { + this(new HashMap<>()); + } + + /** + * Default constructor. The AudioSegmentExport can be configured via named properties in the provided HashMap. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where files should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public AudioSegmentExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + } + + /** + * Processes a SegmentContainer: Extract audio-data and writes to a WAVE file. + * + * @param shot SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer shot) { + try { + /* Prepare folder and OutputStream. */ + Path directory = this.destination.resolve(shot.getSuperId()); + Files.createDirectories(directory); + OutputStream stream = Files.newOutputStream(directory.resolve(shot.getId() + ".wav"), CREATE, TRUNCATE_EXISTING); + + /* Extract mean samples and perpare byte buffer. */ + short[] data = shot.getMeanSamplesAsShort(); + ByteBuffer buffer = ByteBuffer.allocate(44 + data.length * 2).order(ByteOrder.LITTLE_ENDIAN); + + /* Write header of WAV file. */ + this.writeWaveHeader(buffer, shot.getSamplingrate(), (short) 1, data.length); + + /* Write actual data. */ + for (short sample : data) { + buffer.putShort(sample); + } + + stream.write(buffer.array()); + stream.close(); + } catch (IOException | BufferOverflowException e) { + LOGGER.fatal("Could not export audio segment {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(e)); } - - /** - * Processes a SegmentContainer: Extract audio-data and writes to a WAVE file. - * - * @param shot SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer shot) { - try { - /* Prepare folder and OutputStream. */ - Path directory = this.destination.resolve(shot.getSuperId()); - Files.createDirectories(directory); - OutputStream stream = Files.newOutputStream(directory.resolve(shot.getId()+".wav"), CREATE, TRUNCATE_EXISTING); - - /* Extract mean samples and perpare byte buffer. */ - short[] data = shot.getMeanSamplesAsShort(); - ByteBuffer buffer = ByteBuffer.allocate(44 + data.length*2).order(ByteOrder.LITTLE_ENDIAN); - - /* Write header of WAV file. */ - this.writeWaveHeader(buffer, shot.getSamplingrate(), (short)1, data.length); - - /* Write actual data. */ - for (short sample : data) { - buffer.putShort(sample); - } - - stream.write(buffer.array()); - stream.close(); - } catch (IOException | BufferOverflowException e) { - LOGGER.fatal("Could not export audio segment {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(e)); - } - } - - /** - * Writes the WAV header to the ByteBuffer (1 channel). - * - * @param buffer The buffer to which to write the header. - * @param channels The number of channels in the WAV file. - * @param samplingrate Samplingrate of the output file. - * @param length Length in bytes of the frames data - */ - private void writeWaveHeader(ByteBuffer buffer, float samplingrate, short channels, int length) { - /* Length of the subChunk2. */ - final int subChunk2Length = length * channels * (AudioFrame.BITS_PER_SAMPLE/8); /* Number of bytes for audio data: NumSamples * NumChannels * BitsPerSample/8. */ - - /* RIFF Chunk. */ - buffer.put("RIFF".getBytes()); - buffer.putInt(36 + subChunk2Length); - buffer.put("WAVE".getBytes()); /* WAV format. */ - - /* Format chunk. */ - buffer.put("fmt ".getBytes()); /* Begin of the format chunk. */ - buffer.putInt(16); /* Length of the Format chunk. */ - buffer.putShort((short)1); /* Format: 1 = Raw PCM (linear quantization). */ - buffer.putShort((short)1); /* Number of channels. */ - buffer.putInt((int)samplingrate); /* Samplingrate. */ - buffer.putInt((int)(samplingrate * channels * (AudioFrame.BITS_PER_SAMPLE/8))); /* Byte rate: SampleRate * NumChannels * BitsPerSample/8 */ - buffer.putShort((short)(channels * (AudioFrame.BITS_PER_SAMPLE/8))); /* Block align: NumChannels * BitsPerSample/8. */ - buffer.putShort((short)(AudioFrame.BITS_PER_SAMPLE)) /* Bits per sample. */; - - /* Data chunk */ - buffer.put("data".getBytes()); /* Begin of the data chunk. */ - buffer.putInt(subChunk2Length); /* Length of the data chunk. */ - } - - - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Nothing to init. */ } - - @Override - public void finish() { /* Nothing to finish. */} - - @Override - public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } - - @Override - public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */} + } + + /** + * Writes the WAV header to the ByteBuffer (1 channel). + * + * @param buffer The buffer to which to write the header. + * @param channels The number of channels in the WAV file. + * @param samplingrate Samplingrate of the output file. + * @param length Length in bytes of the frames data + */ + private void writeWaveHeader(ByteBuffer buffer, float samplingrate, short channels, int length) { + /* Length of the subChunk2. */ + final int subChunk2Length = length * channels * (AudioFrame.BITS_PER_SAMPLE / 8); /* Number of bytes for audio data: NumSamples * NumChannels * BitsPerSample/8. */ + + /* RIFF Chunk. */ + buffer.put("RIFF".getBytes()); + buffer.putInt(36 + subChunk2Length); + buffer.put("WAVE".getBytes()); /* WAV format. */ + + /* Format chunk. */ + buffer.put("fmt ".getBytes()); /* Begin of the format chunk. */ + buffer.putInt(16); /* Length of the Format chunk. */ + buffer.putShort((short) 1); /* Format: 1 = Raw PCM (linear quantization). */ + buffer.putShort((short) 1); /* Number of channels. */ + buffer.putInt((int) samplingrate); /* Samplingrate. */ + buffer.putInt((int) (samplingrate * channels * (AudioFrame.BITS_PER_SAMPLE / 8))); /* Byte rate: SampleRate * NumChannels * BitsPerSample/8 */ + buffer.putShort((short) (channels * (AudioFrame.BITS_PER_SAMPLE / 8))); /* Block align: NumChannels * BitsPerSample/8. */ + buffer.putShort((short) (AudioFrame.BITS_PER_SAMPLE)) /* Bits per sample. */; + + /* Data chunk */ + buffer.put("data".getBytes()); /* Begin of the data chunk. */ + buffer.putInt(subChunk2Length); /* Length of the data chunk. */ + } + + + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Nothing to init. */ } + + @Override + public void finish() { /* Nothing to finish. */} + + @Override + public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } + + @Override + public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java index 7dfb69861..dbe2f0b45 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java @@ -1,5 +1,14 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -12,106 +21,104 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; import org.vitrivr.cineast.core.util.dsp.visualization.AudioSignalVisualizer; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.function.Supplier; - /** - * Visualizes and exporst the power spectogram (time vs. frequency vs. power) of the provided - * AudioSegment. - * + * Visualizes and exporst the power spectogram (time vs. frequency vs. power) of the provided AudioSegment. */ public class AudioSpectogramExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_WIDTH = "width"; - private static final String PROPERTY_NAME_HEIGHT = "height"; - private static final String PROPERTY_NAME_FORMAT = "format"; - - /** Destination path; can be set in the AudioWaveformExporter properties. */ - private final Path destination; - - /** Width of the resulting image in pixels. */ - private final int width; - - /** Height of the resulting image in pixels. */ - private final int height; - - /** Output format for thumbnails. Defaults to PNG. */ - private final String format; - - /** - * Default constructor - */ - public AudioSpectogramExporter() { - this(new HashMap<>()); - } - - /** - * Constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where images should be stored.
      2. - *
    2. width: Width of the image in pixels.
      3. - *
    3. height: Height of the image in pixels.
      4. - *
    - * - * @param properties HashMap containing named properties - */ - public AudioSpectogramExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); - this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); - this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_WIDTH = "width"; + private static final String PROPERTY_NAME_HEIGHT = "height"; + private static final String PROPERTY_NAME_FORMAT = "format"; + + /** + * Destination path; can be set in the AudioWaveformExporter properties. + */ + private final Path destination; + + /** + * Width of the resulting image in pixels. + */ + private final int width; + + /** + * Height of the resulting image in pixels. + */ + private final int height; + + /** + * Output format for thumbnails. Defaults to PNG. + */ + private final String format; + + /** + * Default constructor + */ + public AudioSpectogramExporter() { + this(new HashMap<>()); + } + + /** + * Constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where images should be stored.
      2. + *
    2. width: Width of the image in pixels.
      3. + *
    3. height: Height of the image in pixels.
      4. + *
    + * + * @param properties HashMap containing named properties + */ + public AudioSpectogramExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); + this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); + this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); + } + + @Override + public void processSegment(SegmentContainer shot) { + /* If shot has no samples, this step is skipped. */ + if (shot.getNumberOfSamples() == 0) { + return; } - @Override - public void processSegment(SegmentContainer shot) { - /* If shot has no samples, this step is skipped. */ - if (shot.getNumberOfSamples() == 0) { - return; - } - - /* Prepare STFT and Spectrum for the segment. */ - final Path directory = this.destination.resolve(shot.getSuperId()); - final STFT stft = shot.getSTFT(2048, 512, new HanningWindow()); - final List spectrums = stft.getPowerSpectrum(); - - /* Visualize Spectrum and write it to disc. */ - try { - BufferedImage image = AudioSignalVisualizer.visualizeSpectogram(spectrums, this.width, this.height); - if (image != null) { - Files.createDirectories(directory); - ImageIO.write(image, format, directory.resolve(shot.getId() + "." + format.toLowerCase()).toFile()); - } else { - LOGGER.warn("Spectrum could not be visualized!"); - } - } catch (IOException exception) { - LOGGER.error("A serious error occurred while writing the spectrum image! ({})", LogHelper.getStackTrace(exception)); - } + /* Prepare STFT and Spectrum for the segment. */ + final Path directory = this.destination.resolve(shot.getSuperId()); + final STFT stft = shot.getSTFT(2048, 512, new HanningWindow()); + final List spectrums = stft.getPowerSpectrum(); + + /* Visualize Spectrum and write it to disc. */ + try { + BufferedImage image = AudioSignalVisualizer.visualizeSpectogram(spectrums, this.width, this.height); + if (image != null) { + Files.createDirectories(directory); + ImageIO.write(image, format, directory.resolve(shot.getId() + "." + format.toLowerCase()).toFile()); + } else { + LOGGER.warn("Spectrum could not be visualized!"); + } + } catch (IOException exception) { + LOGGER.error("A serious error occurred while writing the spectrum image! ({})", LogHelper.getStackTrace(exception)); } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupplier, int batchSize) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupplier, int batchSize) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java index c489ceecd..beb386fcf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java @@ -1,16 +1,7 @@ package org.vitrivr.cineast.core.features.exporter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.frames.AudioFrame; -import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.extractor.Extractor; -import org.vitrivr.cineast.core.util.LogHelper; - -import javax.imageio.ImageIO; -import java.awt.*; +import java.awt.Color; +import java.awt.Graphics; import java.awt.image.BufferedImage; import java.io.IOException; import java.nio.file.Files; @@ -19,131 +10,154 @@ import java.util.HashMap; import java.util.List; import java.util.function.Supplier; +import javax.imageio.ImageIO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.frames.AudioFrame; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.extractor.Extractor; +import org.vitrivr.cineast.core.util.LogHelper; /** * Visualizes and export the audio-waveform of the audio data in a given Segment. - * */ public class AudioWaveformExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_WIDTH = "width"; - private static final String PROPERTY_NAME_HEIGHT = "height"; - private static final String PROPERTY_NAME_FORMAT = "format"; - - /** Destination path; can be set in the AudioWaveformExporter properties. */ - private final Path destination; - - /** Width of the resulting image in pixels. */ - private final int width; - - /** Height of the resulting image in pixels. */ - private final int height; - /** Output format for thumbnails. Defaults to PNG. */ - private final String format; - - /** Background color of the resulting image. */ - private final Color backgroundColor = Color.lightGray; - - /** Foreground color of the resulting image (used to draw the waveform). */ - private final Color foregroundColor = Color.darkGray; - - /** - * Default constructor - */ - public AudioWaveformExporter() { - this(new HashMap<>()); - } + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_WIDTH = "width"; + private static final String PROPERTY_NAME_HEIGHT = "height"; + private static final String PROPERTY_NAME_FORMAT = "format"; + + /** + * Destination path; can be set in the AudioWaveformExporter properties. + */ + private final Path destination; + + /** + * Width of the resulting image in pixels. + */ + private final int width; + + /** + * Height of the resulting image in pixels. + */ + private final int height; + + /** + * Output format for thumbnails. Defaults to PNG. + */ + private final String format; + + /** + * Background color of the resulting image. + */ + private final Color backgroundColor = Color.lightGray; + + /** + * Foreground color of the resulting image (used to draw the waveform). + */ + private final Color foregroundColor = Color.darkGray; + + /** + * Default constructor + */ + public AudioWaveformExporter() { + this(new HashMap<>()); + } + + /** + * Constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where images should be stored.
      2. + *
    2. width: Width of the image in pixels.
      3. + *
    3. height: Height of the image in pixels.
      4. + *
    + * + * @param properties HashMap containing named properties + */ + public AudioWaveformExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "600")); + this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "250")); + this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); + } + + /** + * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. + * + * @param shot SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer shot) { + try { + /* If shot has no samples, this step is skipped. */ + if (shot.getNumberOfSamples() == 0) { + return; + } + + Path directory = this.destination.resolve(shot.getSuperId()); + Files.createDirectories(directory); + + List frames = shot.getAudioFrames(); + + BufferedImage image = new BufferedImage(this.width, this.height, BufferedImage.TYPE_INT_RGB); + Graphics graphics = image.getGraphics(); + graphics.setColor(this.backgroundColor); + graphics.fillRect(0, 0, width, height); + + /* Determine the number of samples. */ + int samples = shot.getNumberOfSamples(); + + /* Determine y position of baseline and x/y ratios. */ + int baseline = this.height / 2; + float bin_width = (float) this.width / (float) samples; + + int s_tot = 0; + + /* Initialize bins (one for every pixel in the x direction). */ + double[] bin_max = new double[this.width]; + double[] bin_min = new double[this.width]; + + for (AudioFrame frame : frames) { + for (int s = 0; s < frame.numberOfSamples(); s++, s_tot++) { + int bin = (int) (bin_width * s_tot); + double amplitude = frame.getMeanSampleAsDouble(s); + bin_max[bin] = Math.max(bin_max[bin], amplitude); + bin_min[bin] = Math.min(bin_min[bin], amplitude); + } + } - /** - * Constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where images should be stored.
      2. - *
    2. width: Width of the image in pixels.
      3. - *
    3. height: Height of the image in pixels.
      4. - *
    - * - * @param properties HashMap containing named properties - */ - public AudioWaveformExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "600")); - this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "250")); - this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); - } + /* Draw all the bins. */ + graphics.setColor(foregroundColor); + for (int i = 1; i < this.width; i++) { + graphics.drawLine(i, baseline + (int) (bin_min[i] * baseline), i, baseline + (int) (bin_max[i] * baseline)); + } - /** - * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. - * - * @param shot SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer shot) { - try { - /* If shot has no samples, this step is skipped. */ - if (shot.getNumberOfSamples() == 0) { - return; - } - - Path directory = this.destination.resolve(shot.getSuperId()); - Files.createDirectories(directory); - - List frames = shot.getAudioFrames(); - - BufferedImage image = new BufferedImage(this.width, this.height, BufferedImage.TYPE_INT_RGB); - Graphics graphics = image.getGraphics(); - graphics.setColor(this.backgroundColor); - graphics.fillRect(0,0,width,height); - - /* Determine the number of samples. */ - int samples = shot.getNumberOfSamples(); - - /* Determine y position of baseline and x/y ratios. */ - int baseline = this.height / 2; - float bin_width = (float) this.width / (float) samples; - - int s_tot = 0; - - /* Initialize bins (one for every pixel in the x direction). */ - double[] bin_max = new double[this.width]; - double[] bin_min = new double[this.width]; - - for (AudioFrame frame : frames) { - for (int s = 0; s < frame.numberOfSamples(); s++, s_tot++) { - int bin = (int) (bin_width * s_tot); - double amplitude = frame.getMeanSampleAsDouble(s); - bin_max[bin] = Math.max(bin_max[bin], amplitude); - bin_min[bin] = Math.min(bin_min[bin], amplitude); - } - } - - /* Draw all the bins. */ - graphics.setColor(foregroundColor); - for (int i = 1; i < this.width; i++) { - graphics.drawLine(i, baseline + (int) (bin_min[i] * baseline), i, baseline + (int) (bin_max[i] * baseline)); - } - - ImageIO.write(image, format, directory.resolve(shot.getId() + "." + format.toLowerCase()).toFile()); - } catch (IOException exception) { - LOGGER.fatal("Could not export waveform image for audio segment {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); - } + ImageIO.write(image, format, directory.resolve(shot.getId() + "." + format.toLowerCase()).toFile()); + } catch (IOException exception) { + LOGGER.fatal("Could not export waveform image for audio segment {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java index 2f22a5f1a..65396b04b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java @@ -1,5 +1,13 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -13,97 +21,96 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; import org.vitrivr.cineast.core.util.dsp.visualization.AudioSignalVisualizer; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.function.Supplier; - public class CENSExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_WIDTH = "width"; - private static final String PROPERTY_NAME_HEIGHT = "height"; - - /** Destination path; can be set in the CENSExporter properties. */ - private final Path destination; - - /** Width of the resulting CENS chromagram image in pixels. */ - private final int width; - - /** Height of the resulting CENS chromagram image in pixels. */ - private final int height; - - - /** - * Default constructor - */ - public CENSExporter() { - this(new HashMap<>()); - } - - /** - * Constructor. The CENSExporter can be configured via named properties in the provided HashMap. Supported parameters: - * - *
      - *
    1. destination: Path where images should be stored.
      2. - *
    2. width: Width of the image in pixels.
      3. - *
    3. height: Height of the image in pixels.
      4. - *
    - * - * @param properties HashMap containing named properties - */ - public CENSExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); - this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_WIDTH = "width"; + private static final String PROPERTY_NAME_HEIGHT = "height"; + + /** + * Destination path; can be set in the CENSExporter properties. + */ + private final Path destination; + + /** + * Width of the resulting CENS chromagram image in pixels. + */ + private final int width; + + /** + * Height of the resulting CENS chromagram image in pixels. + */ + private final int height; + + + /** + * Default constructor + */ + public CENSExporter() { + this(new HashMap<>()); + } + + /** + * Constructor. The CENSExporter can be configured via named properties in the provided HashMap. Supported parameters: + * + *
      + *
    1. destination: Path where images should be stored.
      2. + *
    2. width: Width of the image in pixels.
      3. + *
    3. height: Height of the image in pixels.
      4. + *
    + * + * @param properties HashMap containing named properties + */ + public CENSExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); + this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); + } + + @Override + public void processSegment(SegmentContainer shot) { + /* IF shot has no samples, this step is skipped. */ + if (shot.getNumberOfSamples() == 0) { + return; } - @Override - public void processSegment(SegmentContainer shot) { - /* IF shot has no samples, this step is skipped. */ - if (shot.getNumberOfSamples() == 0) { - return; - } - - /* Prepare STFT and HPCP for the segment. */ - final Path directory = this.destination.resolve(shot.getSuperId()); - final STFT stft = shot.getSTFT(2048, 512, new HanningWindow()); - final HPCP hpcp = new HPCP(); - hpcp.addContribution(stft); - - double[][] cens = CENS.cens(hpcp, 25, 5); - - /* Visualize chromagram and write it to disc. */ - try { - BufferedImage image = AudioSignalVisualizer.visualizeCens(cens, this.width, this.height); - if (image != null) { - Files.createDirectories(directory); - ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); - } else { - LOGGER.warn("CENS chromagram could not be visualized!"); - } - } catch (IOException exception) { - LOGGER.error("A serious error occurred while writing the CENS chromagram image! ({})", LogHelper.getStackTrace(exception)); - } + /* Prepare STFT and HPCP for the segment. */ + final Path directory = this.destination.resolve(shot.getSuperId()); + final STFT stft = shot.getSTFT(2048, 512, new HanningWindow()); + final HPCP hpcp = new HPCP(); + hpcp.addContribution(stft); + + double[][] cens = CENS.cens(hpcp, 25, 5); + + /* Visualize chromagram and write it to disc. */ + try { + BufferedImage image = AudioSignalVisualizer.visualizeCens(cens, this.width, this.height); + if (image != null) { + Files.createDirectories(directory); + ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + } else { + LOGGER.warn("CENS chromagram could not be visualized!"); + } + } catch (IOException exception) { + LOGGER.error("A serious error occurred while writing the CENS chromagram image! ({})", LogHelper.getStackTrace(exception)); } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java index ee7ecf0e5..230992d0a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java @@ -1,5 +1,13 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.Pair; @@ -14,93 +22,91 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; import org.vitrivr.cineast.core.util.dsp.visualization.AudioSignalVisualizer; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.function.Supplier; - public class ChromagramExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_WIDTH = "width"; - private static final String PROPERTY_NAME_HEIGHT = "height"; - - /** Destination path; can be set in the ChromagramExporter properties. */ - private final Path destination; - - /** Width of the resulting chromagram image in pixels. */ - private final int width; - - /** Height of the resulting chromagram image in pixels. */ - private final int height; - - - /** - * Default constructor. The ChromagramExporter can be configured via named properties - * in the provided HashMap. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where images should be stored.
      2. - *
    2. width: Width of the image in pixels.
      3. - *
    3. height: Height of the image in pixels.
      4. - *
    - * - * @param properties HashMap containing named properties - */ - public ChromagramExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); - this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_WIDTH = "width"; + private static final String PROPERTY_NAME_HEIGHT = "height"; + + /** + * Destination path; can be set in the ChromagramExporter properties. + */ + private final Path destination; + + /** + * Width of the resulting chromagram image in pixels. + */ + private final int width; + + /** + * Height of the resulting chromagram image in pixels. + */ + private final int height; + + + /** + * Default constructor. The ChromagramExporter can be configured via named properties in the provided HashMap. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where images should be stored.
      2. + *
    2. width: Width of the image in pixels.
      3. + *
    3. height: Height of the image in pixels.
      4. + *
    + * + * @param properties HashMap containing named properties + */ + public ChromagramExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); + this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); + } + + @Override + public void processSegment(SegmentContainer shot) { + /* IF shot has no samples, this step is skipped. */ + if (shot.getNumberOfSamples() == 0) { + return; } - @Override - public void processSegment(SegmentContainer shot) { - /* IF shot has no samples, this step is skipped. */ - if (shot.getNumberOfSamples() == 0) { - return; - } - - /* Prepare STFT and HPCP for the segment. */ - final Path directory = this.destination.resolve(shot.getSuperId()); - final Pair parameters = FFTUtil.parametersForDuration(shot.getSamplingrate(), 0.1f); - final STFT stft = shot.getSTFT(parameters.first, (parameters.first-2*parameters.second)/2, parameters.second, new HanningWindow()); - final HPCP hpcp = new HPCP(); - hpcp.addContribution(stft); - - /* Visualize chromagram and write it to disc. */ - try { - BufferedImage image = AudioSignalVisualizer.visualizeChromagram(hpcp, this.width, this.height); - if (image != null) { - Files.createDirectories(directory); - ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); - } else { - LOGGER.warn("Chromagram could not be visualized!"); - } - } catch (IOException exception) { - LOGGER.error("A serious error occurred while writing the chromagram image! ({})", LogHelper.getStackTrace(exception)); - } + /* Prepare STFT and HPCP for the segment. */ + final Path directory = this.destination.resolve(shot.getSuperId()); + final Pair parameters = FFTUtil.parametersForDuration(shot.getSamplingrate(), 0.1f); + final STFT stft = shot.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow()); + final HPCP hpcp = new HPCP(); + hpcp.addContribution(stft); + + /* Visualize chromagram and write it to disc. */ + try { + BufferedImage image = AudioSignalVisualizer.visualizeChromagram(hpcp, this.width, this.height); + if (image != null) { + Files.createDirectories(directory); + ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + } else { + LOGGER.warn("Chromagram could not be visualized!"); + } + } catch (IOException exception) { + LOGGER.error("A serious error occurred while writing the chromagram image! ({})", LogHelper.getStackTrace(exception)); } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java index d14ac1fe6..9ced9f960 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java @@ -1,5 +1,10 @@ package org.vitrivr.cineast.core.features.exporter; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.frames.VideoFrame; @@ -9,75 +14,72 @@ import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.util.LogHelper; -import javax.imageio.ImageIO; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.function.Supplier; - public class FrameExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_FORMAT = "format"; - private static final String PROPERTY_NAME_OFFSET = "offset"; - private final File folder; - private final int offset; - private final String format; - + private static final Logger LOGGER = LogManager.getLogger(); + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_FORMAT = "format"; + private static final String PROPERTY_NAME_OFFSET = "offset"; + + private final File folder; + private final int offset; + private final String format; + - /** - * Default constructor - no parameters. - */ - public FrameExporter() { - this(new HashMap<>()); - } + /** + * Default constructor - no parameters. + */ + public FrameExporter() { + this(new HashMap<>()); + } - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where motion frameimages should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public FrameExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./thumbnails")); - this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "PNG"); - this.offset= Integer.valueOf(properties.getOrDefault(PROPERTY_NAME_OFFSET, "1")); - } + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where motion frameimages should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public FrameExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./thumbnails")); + this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "PNG"); + this.offset = Integer.valueOf(properties.getOrDefault(PROPERTY_NAME_OFFSET, "1")); + } - - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - if(!this.folder.exists()){ - this.folder.mkdirs(); - } - } + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + if (!this.folder.exists()) { + this.folder.mkdirs(); + } + } - @Override - public void processSegment(SegmentContainer shot) { - for(VideoFrame f : shot.getVideoFrames()){ - if(f.getId() % this.offset == 0){ - try { - ImageIO.write(f.getImage().getBufferedImage(), this.format, new File(folder, String.format("%06d",(f.getId() / this.offset)) + "." + this.format)); - } catch (IOException e) { - LOGGER.error("Error while exporting frame: {}", LogHelper.getStackTrace(e)); - } - } - } - } + @Override + public void processSegment(SegmentContainer shot) { + for (VideoFrame f : shot.getVideoFrames()) { + if (f.getId() % this.offset == 0) { + try { + ImageIO.write(f.getImage().getBufferedImage(), this.format, new File(folder, String.format("%06d", (f.getId() / this.offset)) + "." + this.format)); + } catch (IOException e) { + LOGGER.error("Error while exporting frame: {}", LogHelper.getStackTrace(e)); + } + } + } + } - @Override - public void finish() {} + @Override + public void finish() { + } - @Override - public void initalizePersistentLayer(Supplier supply) {} + @Override + public void initalizePersistentLayer(Supplier supply) { + } - @Override - public void dropPersistentLayer(Supplier supply) {} + @Override + public void dropPersistentLayer(Supplier supply) { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java index 239073c79..5ce93570f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java @@ -1,5 +1,15 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.Color; +import java.awt.Graphics; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.m3d.WritableMesh; @@ -11,124 +21,127 @@ import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.core.util.mesh.MeshColoringUtil; -import javax.imageio.ImageIO; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.function.Supplier; - public class Model3DThumbnailExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_SIZE = "size"; - - /** List of perspective that should be rendered. Azimuth and polar angles in degree. */ - private static final float[][] PERSPECTIVES = { - {0.0f,90.0f}, - {45.0f,135.0f}, - {-135.0f,-225.0f}, - {0.0f,-90.0f} - }; - - /** Distance of camera from object. */ - private static final float DISTANCE = 2.0f; - - /** Destination path; can be set in the AudioWaveformExporter properties. */ - private final Path destination; - - /** Size of the resulting image in pixels (image will have dimension size x size). */ - private final int size; - - /** Offscreen rendering context. */ - private final JOGLOffscreenRenderer renderer; - - /** Background color of the resulting image. */ - private Color backgroundColor = Color.lightGray; - - /** - * Default constructor. The AudioWaveformExporter can be configured via named properties - * in the provided HashMap. Supported parameters: - * - *
      - *
    1. destination: Path where images should be stored.
      2. - *
    2. size: Width of the resulting image in pixels (size x size).
      3. - *
    - * - * @param properties HashMap containing named properties - */ - public Model3DThumbnailExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.size = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_SIZE, "800")); - this.renderer = new JOGLOffscreenRenderer(this.size/2, this.size/2); - } - /** - * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. - * - * @param shot SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer shot) { - Path directory = this.destination.resolve(shot.getSuperId()); - try { - Files.createDirectories(directory); - WritableMesh mesh = shot.copyNormalizedMesh(); - - if (!mesh.isEmpty()) { - /* Colors the mesh. */ - MeshColoringUtil.normalColoring(mesh); - - BufferedImage buffer = null; - BufferedImage image = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); - Graphics graphics = image.getGraphics(); - - - if (this.renderer.retain()) { - this.renderer.clear(this.backgroundColor); - this.renderer.assemble(mesh); - - for (int i=0; i<4; i++) { - this.renderer.positionCameraPolar( DISTANCE, PERSPECTIVES[i][0], PERSPECTIVES[i][1], 0.0, 0.0, 0.0); - this.renderer.render(); - buffer = this.renderer.obtain(); - - int idx = i % 2; - int idy = i < 2 ? 0 : 1; - int sz = this.size/2; - - graphics.drawImage(buffer, idx * sz, idy*sz, null); - } - } else { - LOGGER.error("Could not export thumbnail image for model {} because renderer could not be retained by current thread.", shot.getId()); - } - ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); - } - } catch (IOException exception) { - LOGGER.fatal("Could not export thumbnail image for model {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); - } catch (Exception exception) { - LOGGER.error("Could not export thumbnail image for model {} because an unknown exception occurred ({}).", shot.getId(), LogHelper.getStackTrace(exception)); - } finally { - this.renderer.release(); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_SIZE = "size"; + + /** + * List of perspective that should be rendered. Azimuth and polar angles in degree. + */ + private static final float[][] PERSPECTIVES = { + {0.0f, 90.0f}, + {45.0f, 135.0f}, + {-135.0f, -225.0f}, + {0.0f, -90.0f} + }; + + /** + * Distance of camera from object. + */ + private static final float DISTANCE = 2.0f; + + /** + * Destination path; can be set in the AudioWaveformExporter properties. + */ + private final Path destination; + + /** + * Size of the resulting image in pixels (image will have dimension size x size). + */ + private final int size; + + /** + * Offscreen rendering context. + */ + private final JOGLOffscreenRenderer renderer; + + /** + * Background color of the resulting image. + */ + private Color backgroundColor = Color.lightGray; + + /** + * Default constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. Supported parameters: + * + *
      + *
    1. destination: Path where images should be stored.
      2. + *
    2. size: Width of the resulting image in pixels (size x size).
      3. + *
    + * + * @param properties HashMap containing named properties + */ + public Model3DThumbnailExporter(HashMap properties) { + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.size = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_SIZE, "800")); + this.renderer = new JOGLOffscreenRenderer(this.size / 2, this.size / 2); + } + + /** + * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. + * + * @param shot SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer shot) { + Path directory = this.destination.resolve(shot.getSuperId()); + try { + Files.createDirectories(directory); + WritableMesh mesh = shot.copyNormalizedMesh(); + + if (!mesh.isEmpty()) { + /* Colors the mesh. */ + MeshColoringUtil.normalColoring(mesh); + + BufferedImage buffer = null; + BufferedImage image = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); + Graphics graphics = image.getGraphics(); + + if (this.renderer.retain()) { + this.renderer.clear(this.backgroundColor); + this.renderer.assemble(mesh); + + for (int i = 0; i < 4; i++) { + this.renderer.positionCameraPolar(DISTANCE, PERSPECTIVES[i][0], PERSPECTIVES[i][1], 0.0, 0.0, 0.0); + this.renderer.render(); + buffer = this.renderer.obtain(); + + int idx = i % 2; + int idy = i < 2 ? 0 : 1; + int sz = this.size / 2; + + graphics.drawImage(buffer, idx * sz, idy * sz, null); + } + } else { + LOGGER.error("Could not export thumbnail image for model {} because renderer could not be retained by current thread.", shot.getId()); } + ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + } + } catch (IOException exception) { + LOGGER.fatal("Could not export thumbnail image for model {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); + } catch (Exception exception) { + LOGGER.error("Could not export thumbnail image for model {} because an unknown exception occurred ({}).", shot.getId(), LogHelper.getStackTrace(exception)); + } finally { + this.renderer.release(); } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java index c8a5dc594..5a517feb0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java @@ -1,15 +1,8 @@ package org.vitrivr.cineast.core.features.exporter; import georegression.struct.point.Point2D_F32; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.frames.VideoFrame; -import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.extractor.Extractor; - -import javax.imageio.ImageIO; -import java.awt.*; +import java.awt.Color; +import java.awt.Graphics2D; import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; @@ -18,101 +11,112 @@ import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; +import javax.imageio.ImageIO; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.extractor.Extractor; public class MotionFrameExporter implements Extractor { - private static final String PROPERTY_NAME_DESTINATION = "destination"; - - private final File folder; - - /** - * Default constructor - no parameters. - */ - public MotionFrameExporter() { - this(new HashMap<>()); - } - - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where motion frameimages should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public MotionFrameExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./motion_frames")); - } - - - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - if(!folder.exists()){ - folder.mkdirs(); - } - } - - @Override - public void processSegment(SegmentContainer shot) { - List>> paths = shot.getPaths(); - for(VideoFrame f : shot.getVideoFrames()){ - File file = new File(folder, String.format("%06d",f.getId()) + ".jpg"); - BufferedImage bimg = f.getImage().getBufferedImage(); - for(Pair> pair : paths){ - draw(bimg, pair.second); - } - try { - ImageIO.write(bimg, "jpg", file); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - } - - private static Color[] colors = new Color[]{ - new Color(255, 0, 0), - new Color(230, 30, 0), - new Color(205, 55, 0), - new Color(180, 80, 0), - new Color(155, 105, 0), - new Color(130, 130, 0), - new Color(105, 155, 0), - new Color(80, 180, 0), - new Color(55, 205, 0), - new Color(30, 230, 0), - new Color(0, 255, 0), - - }; - - //private static BasicStroke fgStroke = new BasicStroke(1f, BasicStroke.CAP_ROUND, BasicStroke.JOIN_ROUND); - - public static void draw(BufferedImage img, LinkedList path){ - int col = 0, width = img.getWidth(), height = img.getHeight(); - if(path.size() > 1){ - Graphics2D g = (Graphics2D) img.getGraphics(); - Iterator iter = path.iterator(); - Point2D_F32 last = iter.next(); - //g.setStroke(fgStroke); - while(iter.hasNext()){ - g.setColor(colors[col]); - col = (col + 1) % colors.length; - Point2D_F32 current = iter.next(); - g.drawLine((int)(last.x * width), (int)(last.y * height),(int)(current.x * width), (int)(current.y * height)); - last = current; - } - } - } - - @Override - public void finish() {} - - @Override - public void initalizePersistentLayer(Supplier supply) {} - - @Override - public void dropPersistentLayer(Supplier supply) {} + + private static final String PROPERTY_NAME_DESTINATION = "destination"; + + private final File folder; + + /** + * Default constructor - no parameters. + */ + public MotionFrameExporter() { + this(new HashMap<>()); + } + + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where motion frameimages should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public MotionFrameExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./motion_frames")); + } + + + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + if (!folder.exists()) { + folder.mkdirs(); + } + } + + @Override + public void processSegment(SegmentContainer shot) { + List>> paths = shot.getPaths(); + for (VideoFrame f : shot.getVideoFrames()) { + File file = new File(folder, String.format("%06d", f.getId()) + ".jpg"); + BufferedImage bimg = f.getImage().getBufferedImage(); + for (Pair> pair : paths) { + draw(bimg, pair.second); + } + try { + ImageIO.write(bimg, "jpg", file); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + + private static Color[] colors = new Color[]{ + new Color(255, 0, 0), + new Color(230, 30, 0), + new Color(205, 55, 0), + new Color(180, 80, 0), + new Color(155, 105, 0), + new Color(130, 130, 0), + new Color(105, 155, 0), + new Color(80, 180, 0), + new Color(55, 205, 0), + new Color(30, 230, 0), + new Color(0, 255, 0), + + }; + + //private static BasicStroke fgStroke = new BasicStroke(1f, BasicStroke.CAP_ROUND, BasicStroke.JOIN_ROUND); + + public static void draw(BufferedImage img, LinkedList path) { + int col = 0, width = img.getWidth(), height = img.getHeight(); + if (path.size() > 1) { + Graphics2D g = (Graphics2D) img.getGraphics(); + Iterator iter = path.iterator(); + Point2D_F32 last = iter.next(); + //g.setStroke(fgStroke); + while (iter.hasNext()) { + g.setColor(colors[col]); + col = (col + 1) % colors.length; + Point2D_F32 current = iter.next(); + g.drawLine((int) (last.x * width), (int) (last.y * height), (int) (current.x * width), (int) (current.y * height)); + last = current; + } + } + } + + @Override + public void finish() { + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + } + + @Override + public void dropPersistentLayer(Supplier supply) { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java index 7d9baef4e..60606e85d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.color.ReadableRGBContainer; @@ -10,78 +16,74 @@ import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.core.util.MotionHistoryImage; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.function.Supplier; - public class MotionHistoryImageExporter implements Extractor { - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_FORMAT = "format"; - private static final Logger LOGGER = LogManager.getLogger(); + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_FORMAT = "format"; + private static final Logger LOGGER = LogManager.getLogger(); + + private final File folder; + private final String format; - private final File folder; - private final String format; + /** + * Default constructor - no parameters. + */ + public MotionHistoryImageExporter() { + this(new HashMap<>()); + } - /** - * Default constructor - no parameters. - */ - public MotionHistoryImageExporter() { - this(new HashMap<>()); - } + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where motion history image images should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public MotionHistoryImageExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./motion_history")); + this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "PNG"); + } - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where motion history image images should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public MotionHistoryImageExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./motion_history")); - this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "PNG"); - } + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + if (!this.folder.exists()) { + this.folder.mkdirs(); + } + } - @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - if(!this.folder.exists()){ - this.folder.mkdirs(); - } - } + @Override + public void processSegment(SegmentContainer shot) { + MotionHistoryImage mhi = MotionHistoryImage.motionHistoryImage(shot, 30, 30); + if (mhi == null) { + return; + } + BufferedImage img = new BufferedImage(mhi.getWidth(), mhi.getHeight(), BufferedImage.TYPE_INT_RGB); + int[] colors = new int[mhi.getIntensities().length]; + for (int i = 0; i < colors.length; ++i) { + int c = mhi.getIntensities()[i] * 2; + colors[i] = ReadableRGBContainer.toIntColor(c, c, c); + } + img.setRGB(0, 0, img.getWidth(), img.getHeight(), colors, 0, img.getWidth()); + try { + ImageIO.write(img, format, new File(folder, String.format("%s.%s", shot.getId(), this.format.toLowerCase()))); + } catch (IOException e) { + LOGGER.error("Error while exporting motion history image: {}", LogHelper.getStackTrace(e)); + } + } - @Override - public void processSegment(SegmentContainer shot) { - MotionHistoryImage mhi = MotionHistoryImage.motionHistoryImage(shot, 30, 30); - if(mhi == null){ - return; - } - BufferedImage img = new BufferedImage(mhi.getWidth(), mhi.getHeight(), BufferedImage.TYPE_INT_RGB); - int[] colors = new int[mhi.getIntensities().length]; - for(int i = 0; i < colors.length; ++i){ - int c = mhi.getIntensities()[i] * 2; - colors[i] = ReadableRGBContainer.toIntColor(c, c, c); - } - img.setRGB(0, 0, img.getWidth(), img.getHeight(), colors, 0, img.getWidth()); - try { - ImageIO.write(img, format, new File(folder, String.format("%s.%s", shot.getId(), this.format.toLowerCase()))); - } catch (IOException e) { - LOGGER.error("Error while exporting motion history image: {}", LogHelper.getStackTrace(e)); - } - } + @Override + public void finish() { + } - @Override - public void finish() {} - - @Override - public void initalizePersistentLayer(Supplier supply) {} + @Override + public void initalizePersistentLayer(Supplier supply) { + } - @Override - public void dropPersistentLayer(Supplier supply) {} + @Override + public void dropPersistentLayer(Supplier supply) { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java index 73e1f9671..57c8bb5a7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java @@ -1,99 +1,99 @@ package org.vitrivr.cineast.core.features.exporter; -import java.util.ArrayList; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.ReadableQueryConfig; -import org.vitrivr.cineast.core.data.score.ScoreElement; -import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.DBSelectorSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.retriever.Retriever; -import org.vitrivr.cineast.core.util.LogHelper; - -import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.function.Supplier; +import javax.imageio.ImageIO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.retriever.Retriever; +import org.vitrivr.cineast.core.util.LogHelper; /** - * This class is used to store the query images (i.e. sketches) which is why it implements retriever functionality. - * It is not actually a retriever in the sense that it returns any results. + * This class is used to store the query images (i.e. sketches) which is why it implements retriever functionality. It is not actually a retriever in the sense that it returns any results. */ public class QueryImageExporter implements Retriever { - private static final Logger LOGGER = LogManager.getLogger(); - private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final Logger LOGGER = LogManager.getLogger(); + private static final String PROPERTY_NAME_DESTINATION = "destination"; - private final File folder; - private final DateFormat df = new SimpleDateFormat("dd-MM-yyyy_HH-mm-ss-SSS"); + private final File folder; + private final DateFormat df = new SimpleDateFormat("dd-MM-yyyy_HH-mm-ss-SSS"); - /** - * Default constructor - no parameters. - */ - public QueryImageExporter() { - this(new HashMap<>()); - } + /** + * Default constructor - no parameters. + */ + public QueryImageExporter() { + this(new HashMap<>()); + } - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where query images should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public QueryImageExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./query_images")); - } + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where query images should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public QueryImageExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./query_images")); + } - @Override - public void init(DBSelectorSupplier supply) { - if(!this.folder.exists() || !this.folder.isDirectory()) { - this.folder.mkdirs(); - } - } + @Override + public void init(DBSelectorSupplier supply) { + if (!this.folder.exists() || !this.folder.isDirectory()) { + this.folder.mkdirs(); + } + } - @Override - public void finish() { - } + @Override + public void finish() { + } - @Override - public List getTableNames() { - return new ArrayList<>(); - } + @Override + public List getTableNames() { + return new ArrayList<>(); + } - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - BufferedImage bimg = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); - try { - String filename = (qc != null && qc.getQueryId() != null) ? qc.getQueryId().toString() : this.df.format(Calendar.getInstance().getTime()); - ImageIO.write(bimg, "PNG", new File(folder, filename + ".png")); - } catch (IOException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - } - return Collections.emptyList(); - } + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + BufferedImage bimg = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + try { + String filename = (qc != null && qc.getQueryId() != null) ? qc.getQueryId().toString() : this.df.format(Calendar.getInstance().getTime()); + ImageIO.write(bimg, "PNG", new File(folder, filename + ".png")); + } catch (IOException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + } + return Collections.emptyList(); + } - @Override - public List getSimilar(String segmentId, ReadableQueryConfig qc) { - return Collections.emptyList(); - } + @Override + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + return Collections.emptyList(); + } - @Override - public void initalizePersistentLayer(Supplier supply) {} + @Override + public void initalizePersistentLayer(Supplier supply) { + } - @Override - public void dropPersistentLayer(Supplier supply) {} + @Override + public void dropPersistentLayer(Supplier supply) { + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java index 38ce7b0eb..789d023b3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java @@ -2,6 +2,11 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.frames.VideoFrame; @@ -13,82 +18,76 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.extractor.Extractor; -import javax.imageio.ImageIO; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.function.Supplier; - public class RepresentativeFrameExporter implements Extractor { - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final Logger LOGGER = LogManager.getLogger(); + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final Logger LOGGER = LogManager.getLogger(); + + @SuppressWarnings("rawtypes") + private PersistencyWriter phandler; + private final File folder; - @SuppressWarnings("rawtypes") - private PersistencyWriter phandler; - private final File folder; + /** + * Default constructor - no parameters. + */ + public RepresentativeFrameExporter() { + this(new HashMap<>()); + } - /** - * Default constructor - no parameters. - */ - public RepresentativeFrameExporter() { - this(new HashMap<>()); - } + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where representative frames should be stored.
      2. + *
    + * + * @param properties HashMap containing named properties + */ + public RepresentativeFrameExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./representative_frames")); + } - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where representative frames should be stored.
      2. - *
    - * - * @param properties HashMap containing named properties - */ - public RepresentativeFrameExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./representative_frames")); - } + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + this.phandler = supply.get(); + this.phandler.open("cineast_representativeframes"); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "frame"); + this.folder.mkdirs(); + } - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - this.phandler = supply.get(); - this.phandler.open("cineast_representativeframes"); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "frame"); - this.folder.mkdirs(); - } + @Override + public void processSegment(SegmentContainer segment) { + File outFolder = new File(this.folder, segment.getSuperId()); + outFolder.mkdirs(); + File outFile = new File(outFolder, segment.getId() + ".png"); + VideoFrame f = segment.getMostRepresentativeFrame(); + try { + ImageIO.write(f.getImage().getBufferedImage(), "PNG", outFile); + } catch (IOException e) { + LOGGER.error("Could not write representative frame: {}", e.getMessage()); + } + persist(segment.getId(), f.getId()); + } - @Override - public void processSegment(SegmentContainer segment) { - File outFolder = new File(this.folder, segment.getSuperId()); - outFolder.mkdirs(); - File outFile = new File(outFolder, segment.getId() + ".png"); - VideoFrame f = segment.getMostRepresentativeFrame(); - try { - ImageIO.write(f.getImage().getBufferedImage(), "PNG", outFile); - } catch (IOException e) { - LOGGER.error("Could not write representative frame: {}", e.getMessage()); - } - persist(segment.getId(), f.getId()); - } + protected void persist(String shotId, int frameId) { + this.phandler.persist(this.phandler.generateTuple(shotId, frameId)); + } - protected void persist(String shotId, int frameId) { - this.phandler.persist(this.phandler.generateTuple(shotId, frameId)); - } - - @Override - public void finish() { - this.phandler.close(); - } + @Override + public void finish() { + this.phandler.close(); + } - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createIdEntity("cineast_representativeframes", new AttributeDefinition("frame", AttributeType.INT)); - } + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createIdEntity("cineast_representativeframes", new AttributeDefinition("frame", AttributeType.INT)); + } - @Override - public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity("cineast_representativeframes"); - } + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity("cineast_representativeframes"); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java index b86c3e319..5a4bf7e50 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -8,84 +14,78 @@ import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.util.LogHelper; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.function.Supplier; - public class ShotDescriptorExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - private static final String PROPERTY_NAME_DESTINATION = "destination"; - - private final File folder; - - /** - * Default constructor - no parameters. - */ - public ShotDescriptorExporter() { - this(new HashMap<>()); - } - - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where descriptors should be stored.
      2. - *
    2. format: The image format to use (PNG, JPEG).
      3. - *
    - * - * @param properties HashMap containing named properties - */ - public ShotDescriptorExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./descriptors")); - } - - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - if(!folder.exists()){ - folder.mkdirs(); - } - } - - @Override - public void processSegment(SegmentContainer shot) { - String id = String.format("%06d",shot.getId()); - - BufferedImage img = shot.getAvgImg().getBufferedImage(); - - try { - ImageIO.write(img, "PNG", new File(folder, id + "_avg.png")); - } catch (IOException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - } - - img = shot.getMedianImg().getBufferedImage(); - - try { - ImageIO.write(img, "PNG", new File(folder, id + "_med.png")); - } catch (IOException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - } - - img = shot.getMostRepresentativeFrame().getImage().getBufferedImage(); - - try { - ImageIO.write(img, "PNG", new File(folder, id + "_rep.png")); - } catch (IOException e) { - LOGGER.error(LogHelper.getStackTrace(e)); - } - } - - @Override - public void finish() { /* Nothing to finish. */ } - - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to init. */ } - - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */ } + + private static final Logger LOGGER = LogManager.getLogger(); + private static final String PROPERTY_NAME_DESTINATION = "destination"; + + private final File folder; + + /** + * Default constructor - no parameters. + */ + public ShotDescriptorExporter() { + this(new HashMap<>()); + } + + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where descriptors should be stored.
      2. + *
    2. format: The image format to use (PNG, JPEG).
      3. + *
    + * + * @param properties HashMap containing named properties + */ + public ShotDescriptorExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./descriptors")); + } + + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + if (!folder.exists()) { + folder.mkdirs(); + } + } + + @Override + public void processSegment(SegmentContainer shot) { + String id = String.format("%06d", shot.getId()); + + BufferedImage img = shot.getAvgImg().getBufferedImage(); + + try { + ImageIO.write(img, "PNG", new File(folder, id + "_avg.png")); + } catch (IOException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + } + + img = shot.getMedianImg().getBufferedImage(); + + try { + ImageIO.write(img, "PNG", new File(folder, id + "_med.png")); + } catch (IOException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + } + + img = shot.getMostRepresentativeFrame().getImage().getBufferedImage(); + + try { + ImageIO.write(img, "PNG", new File(folder, id + "_rep.png")); + } catch (IOException e) { + LOGGER.error(LogHelper.getStackTrace(e)); + } + } + + @Override + public void finish() { /* Nothing to finish. */ } + + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to init. */ } + + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */ } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java index 722b125ab..e8b0340da 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.features.exporter; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.function.Supplier; +import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.frames.VideoFrame; @@ -9,94 +15,92 @@ import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.util.LogHelper; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.function.Supplier; - public class ShotThumbnailsExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_FORMAT = "format"; + private static final Logger LOGGER = LogManager.getLogger(); + + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_FORMAT = "format"; + + /** + * Destination folder. + */ + private final File folder; + + /** + * Output format for thumbnails. Defaults to PNG. + */ + private final String format; - /** Destination folder. */ - private final File folder; + /** + * Default constructor - no parameters. + */ + public ShotThumbnailsExporter() { + this(new HashMap<>()); + } - /** Output format for thumbnails. Defaults to PNG. */ - private final String format; + /** + * Constructor with property HashMap that allows for passing of parameters. + *

    + * Supported parameters: + * + *

      + *
    1. destination: Path where shot thumbnails should be stored.
      2. + *
    2. format: The image format to use (PNG, JPEG).
      3. + *
    + * + * @param properties HashMap containing named properties + */ + public ShotThumbnailsExporter(HashMap properties) { + this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./thumbnails")); + this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); + } - /** - * Default constructor - no parameters. - */ - public ShotThumbnailsExporter() { - this(new HashMap<>()); - } + @Override + public void init(PersistencyWriterSupplier supply, int batchSize) { + if (!this.folder.exists()) { + this.folder.mkdirs(); + } + } - /** - * Constructor with property HashMap that allows for passing of parameters. - * - * Supported parameters: - * - *
      - *
    1. destination: Path where shot thumbnails should be stored.
      2. - *
    2. format: The image format to use (PNG, JPEG).
      3. - *
    - * - * @param properties HashMap containing named properties - */ - public ShotThumbnailsExporter(HashMap properties) { - this.folder = new File(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./thumbnails")); - this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); - } + @Override + public void processSegment(SegmentContainer shot) { - @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - if(!this.folder.exists()){ - this.folder.mkdirs(); - } - } + File imageFolder = new File(this.folder, shot.getSuperId()); + File img = new File(imageFolder, shot.getId() + "." + this.format.toLowerCase()); + if (img.exists()) { + return; + } + VideoFrame mostRepresentativeFrame = shot.getMostRepresentativeFrame(); + if (mostRepresentativeFrame == VideoFrame.EMPTY_VIDEO_FRAME) { + return; + } + BufferedImage thumb = mostRepresentativeFrame.getImage().getThumbnailImage(); + try { + if (!imageFolder.exists()) { + imageFolder.mkdirs(); + } + boolean writeSuccess = ImageIO.write(thumb, format, img); + if (!writeSuccess) { + LOGGER.warn("Could not find appropriate writer for thumbnail \"{}\", attempting conversion.", shot.getId()); + BufferedImage convertedThumb = new BufferedImage(thumb.getWidth(), thumb.getHeight(), BufferedImage.TYPE_INT_RGB); + convertedThumb.getGraphics().drawImage(thumb, 0, 0, null); + writeSuccess = ImageIO.write(convertedThumb, format, img); + if (!writeSuccess) { + LOGGER.error("Could not find appropriate writer for thumbnail \"{}\", even after conversion!", shot.getId()); + } + } + } catch (IOException e) { + LOGGER.error("Could not write thumbnail image: {}", LogHelper.getStackTrace(e)); + } + } - @Override - public void processSegment(SegmentContainer shot) { - - File imageFolder = new File(this.folder, shot.getSuperId()); - File img = new File(imageFolder, shot.getId() + "." + this.format.toLowerCase()); - if(img.exists()){ - return; - } - VideoFrame mostRepresentativeFrame = shot.getMostRepresentativeFrame(); - if (mostRepresentativeFrame == VideoFrame.EMPTY_VIDEO_FRAME) { - return; - } - BufferedImage thumb = mostRepresentativeFrame.getImage().getThumbnailImage(); - try { - if(!imageFolder.exists()){ - imageFolder.mkdirs(); - } - boolean writeSuccess = ImageIO.write(thumb, format, img); - if (!writeSuccess) { - LOGGER.warn("Could not find appropriate writer for thumbnail \"{}\", attempting conversion.", shot.getId()); - BufferedImage convertedThumb = new BufferedImage(thumb.getWidth(), thumb.getHeight(), BufferedImage.TYPE_INT_RGB); - convertedThumb.getGraphics().drawImage(thumb, 0, 0, null); - writeSuccess = ImageIO.write(convertedThumb, format, img); - if (!writeSuccess) { - LOGGER.error("Could not find appropriate writer for thumbnail \"{}\", even after conversion!", shot.getId()); - } - } - } catch (IOException e) { - LOGGER.error("Could not write thumbnail image: {}", LogHelper.getStackTrace(e)); - } - } + @Override + public void finish() { /* Nothing to finish. */ } - @Override - public void finish() { /* Nothing to finish. */ } - - @Override - public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } + @Override + public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } - @Override - public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */ } + @Override + public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */ } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java index 01239272a..7c2c76eaf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java @@ -5,9 +5,10 @@ import org.vitrivr.cineast.core.db.PersistentOperator; public interface Extractor extends PersistentOperator { - void init(PersistencyWriterSupplier phandlerSupply, int batchSize); - void processSegment(SegmentContainer shot); - - void finish(); + void init(PersistencyWriterSupplier phandlerSupply, int batchSize); + + void processSegment(SegmentContainer shot); + + void finish(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/ImageCropper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/ImageCropper.java index 4659cc192..5cb4cb880 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/ImageCropper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/ImageCropper.java @@ -1,10 +1,9 @@ package org.vitrivr.cineast.core.features.neuralnet; -import net.coobird.thumbnailator.Thumbnails; -import net.coobird.thumbnailator.geometry.Positions; - import java.awt.image.BufferedImage; import java.io.IOException; +import net.coobird.thumbnailator.Thumbnails; +import net.coobird.thumbnailator.geometry.Positions; /** * Hides the current implementation of resizing & cropping an image @@ -13,16 +12,14 @@ */ public class ImageCropper { - /** - * Scale an Image to specified x*y parameters - * First scales the image to whichever is smaller, x or y - * then crops the image by cutting off from the center - */ - public static BufferedImage scaleAndCropImage(BufferedImage img, int x, int y) { - try { - return Thumbnails.of(img).size(x, y).crop(Positions.CENTER).asBufferedImage(); - } catch (IOException e) { - throw new RuntimeException(e); - } + /** + * Scale an Image to specified x*y parameters First scales the image to whichever is smaller, x or y then crops the image by cutting off from the center + */ + public static BufferedImage scaleAndCropImage(BufferedImage img, int x, int y) { + try { + return Thumbnails.of(img).size(x, y).crop(Positions.CENTER).asBufferedImage(); + } catch (IOException e) { + throw new RuntimeException(e); } + } } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/GraphHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/GraphHelper.java index 2f6033fde..b0ecac7cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/GraphHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/GraphHelper.java @@ -1,28 +1,28 @@ package org.vitrivr.cineast.core.features.neuralnet.tf; -import org.tensorflow.Graph; - import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.tensorflow.Graph; public final class GraphHelper { - private GraphHelper(){} + private GraphHelper() { + } /** * filters operations which are not part of a provided graph */ - public static List filterOperations(List operations, Graph graph){ + public static List filterOperations(List operations, Graph graph) { - if(operations == null || operations.isEmpty() || graph == null){ + if (operations == null || operations.isEmpty() || graph == null) { return Collections.emptyList(); } ArrayList _return = new ArrayList<>(operations.size()); - for(String operation : operations){ - if(graph.operation(operation) != null){ + for (String operation : operations) { + if (graph.operation(operation) != null) { _return.add(operation); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java index ef70a6b63..382eac6f1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java @@ -1,6 +1,13 @@ package org.vitrivr.cineast.core.features.neuralnet.tf.models; import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.nio.FloatBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import org.tensorflow.Graph; import org.tensorflow.Output; import org.tensorflow.Session; @@ -16,14 +23,6 @@ import org.vitrivr.cineast.core.features.neuralnet.tf.GraphHelper; import org.vitrivr.cineast.core.util.LogHelper; -import java.io.IOException; -import java.nio.FloatBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - public class Inception5h implements AutoCloseable { public static final String[] LABELS = { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java index e03badae3..1ac5d4e5d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java @@ -2,6 +2,13 @@ import com.google.protobuf.InvalidProtocolBufferException; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.tensorflow.Graph; import org.tensorflow.Session; import org.tensorflow.ndarray.Shape; @@ -10,12 +17,6 @@ import org.tensorflow.types.TUint8; import org.vitrivr.cineast.core.util.LogHelper; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - public class DeepLab implements AutoCloseable { private final Graph graph; @@ -49,7 +50,6 @@ public synchronized int[][] processImage(TUint8 input) { TInt64 result = (TInt64) session.runner().feed("ImageTensor", input).fetch("SemanticPredictions").run().get(0); - int w = (int) result.shape().size(2); int h = (int) result.shape().size(1); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLabLabel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLabLabel.java index 8bdbd1ec2..719736f43 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLabLabel.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLabLabel.java @@ -2,7 +2,6 @@ import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.map.hash.TObjectIntHashMap; - import java.util.Collection; public enum DeepLabLabel { @@ -265,14 +264,14 @@ public static DeepLabLabel getDominantLabel(Collection labels) { TObjectIntHashMap hist = new TObjectIntHashMap<>(); - for(DeepLabLabel label : labels){ - hist.adjustOrPutValue(label, 1, 1); + for (DeepLabLabel label : labels) { + hist.adjustOrPutValue(label, 1, 1); } int max = 0; DeepLabLabel dominant = NOTHING; - for(DeepLabLabel label : hist.keySet()){ - if(hist.get(label) > max){ + for (DeepLabLabel label : hist.keySet()) { + if (hist.get(label) > max) { max = hist.get(label); dominant = label; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java index c292eadca..414932f83 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java @@ -2,6 +2,13 @@ import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.PriorityQueue; import org.apache.commons.math3.analysis.function.Sigmoid; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -24,14 +31,6 @@ import org.vitrivr.cineast.core.util.MathHelper; import org.vitrivr.cineast.core.util.MathHelper.ArgMaxResult; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.PriorityQueue; - /** * based on https://github.com/szaza/tensorflow-example-java @@ -127,8 +126,7 @@ private int getOutputSizeByShape(TFloat32 result) { /** * It classifies the object/objects on the image * - * @param tensorFlowOutput output from the TensorFlow, it is a 13x13x((num_class +1) * 5) tensor - * 125 = (numClass + Tx, Ty, Tw, Th, To) * 5 - cause we have 5 boxes per each cell + * @param tensorFlowOutput output from the TensorFlow, it is a 13x13x((num_class +1) * 5) tensor 125 = (numClass + Tx, Ty, Tw, Th, To) * 5 - cause we have 5 boxes per each cell * @param labels a string vector with the labels * @return a list of recognition objects */ @@ -155,7 +153,7 @@ private List classifyImage(final float[] tensorFlowOutput, final St } private BoundingBox getModel(final float[] tensorFlowOutput, int cx, int cy, int b, int numClass, - int offset) { + int offset) { BoundingBox model = new BoundingBox(); Sigmoid sigmoid = new Sigmoid(); model.setX((cx + sigmoid.value(tensorFlowOutput[offset])) * 32); @@ -174,8 +172,8 @@ private BoundingBox getModel(final float[] tensorFlowOutput, int cx, int cy, int } private void calculateTopPredictions(final BoundingBox boundingBox, - final PriorityQueue predictionQueue, - final String[] labels) { + final PriorityQueue predictionQueue, + final String[] labels) { for (int i = 0; i < boundingBox.getClasses().length; i++) { ArgMaxResult argMax = MathHelper.argMax(MathHelper.softmax(boundingBox.getClasses())); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoundingBox.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoundingBox.java index e63fce60d..f67553376 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoundingBox.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoundingBox.java @@ -4,58 +4,59 @@ * Model to store the data of a bounding box */ public class BoundingBox { - private double x; - private double y; - private double width; - private double height; - private double confidence; - private double[] classes; - - public double getX() { - return x; - } - - public void setX(double x) { - this.x = x; - } - - public double getY() { - return y; - } - - public void setY(double y) { - this.y = y; - } - - public double getWidth() { - return width; - } - - public void setWidth(double width) { - this.width = width; - } - - public double getHeight() { - return height; - } - - public void setHeight(double height) { - this.height = height; - } - - public double getConfidence() { - return confidence; - } - - public void setConfidence(double confidence) { - this.confidence = confidence; - } - - public double[] getClasses() { - return classes; - } - - public void setClasses(double[] classes) { - this.classes = classes; - } + + private double x; + private double y; + private double width; + private double height; + private double confidence; + private double[] classes; + + public double getX() { + return x; + } + + public void setX(double x) { + this.x = x; + } + + public double getY() { + return y; + } + + public void setY(double y) { + this.y = y; + } + + public double getWidth() { + return width; + } + + public void setWidth(double width) { + this.width = width; + } + + public double getHeight() { + return height; + } + + public void setHeight(double height) { + this.height = height; + } + + public double getConfidence() { + return confidence; + } + + public void setConfidence(double confidence) { + this.confidence = confidence; + } + + public double[] getClasses() { + return classes; + } + + public void setClasses(double[] classes) { + this.classes = classes; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoxPosition.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoxPosition.java index 09170b94b..71d7ecdb2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoxPosition.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/BoxPosition.java @@ -4,114 +4,115 @@ * Model to store the position of the bounding boxes */ public class BoxPosition { - private float left; - private float top; - private float right; - private float bottom; - private float width; - private float height; - - public BoxPosition(float left, float top, float width, float height) { - this.left = left; - this.top = top; - this.width = width; - this.height = height; - - init(); - } - - public BoxPosition(final BoxPosition boxPosition) { - this.left = boxPosition.left; - this.top = boxPosition.top; - this.width = boxPosition.width; - this.height = boxPosition.height; - - init(); - } - - public BoxPosition(final BoxPosition boxPosition, final float scaleX, final float scaleY) { - this.left = boxPosition.left * scaleX; - this.top = boxPosition.top * scaleY; - this.width = boxPosition.width * scaleX; - this.height = boxPosition.height * scaleY; - - init(); - } - - public void init() { - float tmpLeft = this.left; - float tmpTop = this.top; - float tmpRight = this.left + this.width; - float tmpBottom = this.top + this.height; - - this.left = Math.min(tmpLeft, tmpRight); // left should have lower value as right - this.top = Math.min(tmpTop, tmpBottom); // top should have lower value as bottom - this.right = Math.max(tmpLeft, tmpRight); - this.bottom = Math.max(tmpTop, tmpBottom); - } - - public static boolean overlaps(BoxPosition primary, BoxPosition secondary) { - return primary.getLeft() < secondary.getRight() && primary.getRight() > secondary.getLeft() - && primary.getTop() < secondary.getBottom() && primary.getBottom() > secondary.getTop(); - } - - public float getLeft() { - return left; - } - - public int getLeftInt() { - return (int) left; - } - - public float getTop() { - return top; - } - - public int getTopInt() { - return (int) top; - } - - public float getWidth() { - return width; - } - - public int getWidthInt() { - return (int) width; - } - - public float getHeight() { - return height; - } - - public int getHeightInt() { - return (int) height; - } - - public float getRight() { - return right; - } - - public int getRightInt() { - return (int) right; - } - - public float getBottom() { - return bottom; - } - - public int getBottomInt() { - return (int) bottom; - } - - @Override - public String toString() { - return "BoxPosition{" + - "left=" + left + - ", top=" + top + - ", width=" + width + - ", height=" + height + - '}'; - } + + private float left; + private float top; + private float right; + private float bottom; + private float width; + private float height; + + public BoxPosition(float left, float top, float width, float height) { + this.left = left; + this.top = top; + this.width = width; + this.height = height; + + init(); + } + + public BoxPosition(final BoxPosition boxPosition) { + this.left = boxPosition.left; + this.top = boxPosition.top; + this.width = boxPosition.width; + this.height = boxPosition.height; + + init(); + } + + public BoxPosition(final BoxPosition boxPosition, final float scaleX, final float scaleY) { + this.left = boxPosition.left * scaleX; + this.top = boxPosition.top * scaleY; + this.width = boxPosition.width * scaleX; + this.height = boxPosition.height * scaleY; + + init(); + } + + public void init() { + float tmpLeft = this.left; + float tmpTop = this.top; + float tmpRight = this.left + this.width; + float tmpBottom = this.top + this.height; + + this.left = Math.min(tmpLeft, tmpRight); // left should have lower value as right + this.top = Math.min(tmpTop, tmpBottom); // top should have lower value as bottom + this.right = Math.max(tmpLeft, tmpRight); + this.bottom = Math.max(tmpTop, tmpBottom); + } + + public static boolean overlaps(BoxPosition primary, BoxPosition secondary) { + return primary.getLeft() < secondary.getRight() && primary.getRight() > secondary.getLeft() + && primary.getTop() < secondary.getBottom() && primary.getBottom() > secondary.getTop(); + } + + public float getLeft() { + return left; + } + + public int getLeftInt() { + return (int) left; + } + + public float getTop() { + return top; + } + + public int getTopInt() { + return (int) top; + } + + public float getWidth() { + return width; + } + + public int getWidthInt() { + return (int) width; + } + + public float getHeight() { + return height; + } + + public int getHeightInt() { + return (int) height; + } + + public float getRight() { + return right; + } + + public int getRightInt() { + return (int) right; + } + + public float getBottom() { + return bottom; + } + + public int getBottomInt() { + return (int) bottom; + } + + @Override + public String toString() { + return "BoxPosition{" + + "left=" + left + + ", top=" + top + + ", width=" + width + + ", height=" + height + + '}'; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Recognition.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Recognition.java index 3e4af4c36..31c42f67c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Recognition.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Recognition.java @@ -4,54 +4,54 @@ * An immutable result returned by a recognizer describing what was recognized. */ public final class Recognition { - /** - * A unique identifier for what has been recognized. Specific to the class, not the instance of - * the object. - */ - private final Integer id; - private final String title; - private final Float confidence; - private BoxPosition location; - - public Recognition(final Integer id, final String title, - final Float confidence, final BoxPosition location) { - this.id = id; - this.title = title; - this.confidence = confidence; - this.location = location; - } - - public Integer getId() { - return id; - } - - public String getTitle() { - return title; - } - - public Float getConfidence() { - return confidence; - } - - public BoxPosition getScaledLocation(final float scaleX, final float scaleY) { - return new BoxPosition(location, scaleX, scaleY); - } - - public BoxPosition getLocation() { - return new BoxPosition(location); - } - - public void setLocation(BoxPosition location) { - this.location = location; - } - - @Override - public String toString() { - return "Recognition{" + - "id=" + id + - ", title='" + title + '\'' + - ", confidence=" + confidence + - ", location=" + location + - '}'; - } + + /** + * A unique identifier for what has been recognized. Specific to the class, not the instance of the object. + */ + private final Integer id; + private final String title; + private final Float confidence; + private BoxPosition location; + + public Recognition(final Integer id, final String title, + final Float confidence, final BoxPosition location) { + this.id = id; + this.title = title; + this.confidence = confidence; + this.location = location; + } + + public Integer getId() { + return id; + } + + public String getTitle() { + return title; + } + + public Float getConfidence() { + return confidence; + } + + public BoxPosition getScaledLocation(final float scaleX, final float scaleY) { + return new BoxPosition(location, scaleX, scaleY); + } + + public BoxPosition getLocation() { + return new BoxPosition(location); + } + + public void setLocation(BoxPosition location) { + this.location = location; + } + + @Override + public String toString() { + return "Recognition{" + + "id=" + id + + ", title='" + title + '\'' + + ", confidence=" + confidence + + ", location=" + location + + '}'; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Size.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Size.java index f4dbe1241..66c15a019 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Size.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/util/Size.java @@ -7,9 +7,7 @@ */ public class Size { - public static final Comparator SIZE_COMPARATOR = new Comparator() - - { + public static final Comparator SIZE_COMPARATOR = new Comparator() { @Override public int compare(final Size lhs, final Size rhs) { // We cast here to ensure the multiplications won't overflow diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/CollectionBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/CollectionBooleanRetriever.java index cddc79972..36bca615f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/CollectionBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/CollectionBooleanRetriever.java @@ -1,10 +1,13 @@ package org.vitrivr.cineast.core.features.retriever; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.features.abstracts.BooleanRetriever; -import java.util.*; - public class CollectionBooleanRetriever extends BooleanRetriever { private static final List SUPPORTED_OPERATORS = diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/MultipleInstantiatableRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/MultipleInstantiatableRetriever.java index bd114cae5..b5313fdd8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/MultipleInstantiatableRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/MultipleInstantiatableRetriever.java @@ -1,13 +1,13 @@ package org.vitrivr.cineast.core.features.retriever; /** - * A marker interface to allow multiple instances of such a retriever. - * These have to ensure that they are distinguishable by {@see #hashCode} and {@see equals}. + * A marker interface to allow multiple instances of such a retriever. These have to ensure that they are distinguishable by {@see #hashCode} and {@see equals}. */ -public interface MultipleInstantiatableRetriever extends Retriever{ +public interface MultipleInstantiatableRetriever extends Retriever { - int hashCode(); - boolean equals(Object o); + int hashCode(); + + boolean equals(Object o); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java index a97a9162a..9cc55ebcf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java @@ -1,29 +1,30 @@ package org.vitrivr.cineast.core.features.retriever; -import com.jogamp.common.util.ArrayHashSet; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.DBSelectorSupplier; import org.vitrivr.cineast.core.db.PersistentOperator; -import java.util.ArrayList; -import java.util.List; - public interface Retriever extends PersistentOperator { - void init(DBSelectorSupplier selectorSupply); - List getSimilar(SegmentContainer sc, ReadableQueryConfig qc); - - List getSimilar(String segmentId, ReadableQueryConfig qc); + void init(DBSelectorSupplier selectorSupply); + + List getSimilar(SegmentContainer sc, ReadableQueryConfig qc); + + List getSimilar(String segmentId, ReadableQueryConfig qc); + + default List getSimilar(List segmentIds, ReadableQueryConfig qc) { + List _return = new ArrayList<>(); + for (String id : segmentIds) { + _return.addAll(getSimilar(id, qc)); + } + return _return; + } - default List getSimilar(List segmentIds, ReadableQueryConfig qc){ - List _return = new ArrayList<>(); - for (String id : segmentIds) { - _return.addAll(getSimilar(id, qc)); - } - return _return; - }; + ; - void finish(); + void finish(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/RetrieverInitializer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/RetrieverInitializer.java index 16e5181cc..e3a14f65f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/RetrieverInitializer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/RetrieverInitializer.java @@ -2,6 +2,6 @@ public interface RetrieverInitializer { - void initialize(Retriever r); - + void initialize(Retriever r); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/IIIFItem.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/IIIFItem.java index 3da975e56..f004150a3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/IIIFItem.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/IIIFItem.java @@ -4,7 +4,9 @@ import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -/** Data class to hold the JSON configuration parameters of a single Image API request */ +/** + * Data class to hold the JSON configuration parameters of a single Image API request + */ public class IIIFItem { @JsonProperty diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/BaseImageRequestValidators.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/BaseImageRequestValidators.java index 3fb0500e7..9962ea5e8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/BaseImageRequestValidators.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/BaseImageRequestValidators.java @@ -74,7 +74,9 @@ public static boolean validateBothWidthAndHeightNotOverridable(boolean isWidthOv return true; } - /** Validates that the float value of a percentage is greater than zero */ + /** + * Validates that the float value of a percentage is greater than zero + */ public static boolean validatePercentageValueGreaterThanZero(float n) { if (n <= 0) { throw new IllegalArgumentException("Percentage value has to be greater than 0"); @@ -82,7 +84,9 @@ public static boolean validatePercentageValueGreaterThanZero(float n) { return true; } - /** Validates that the degree of rotation is >= 0 and < 360 */ + /** + * Validates that the degree of rotation is >= 0 and < 360 + */ public static boolean validateRotationDegrees(float degree) { if (degree < 0 || degree > 360) { throw new IllegalArgumentException("Rotation value can only be between 0° and 360°!"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageFactory.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageFactory.java index 892ba1056..d8868fe28 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageFactory.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageFactory.java @@ -56,7 +56,9 @@ public ImageFactory(Canvas canvas, ImageMetadata globalMetadata) { this.iiifConfig = null; } - /** Helper method to check that given string is not empty */ + /** + * Helper method to check that given string is not empty + */ private static boolean isParamStringValid(String input) { return input != null && input.length() != 0; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageMetadata.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageMetadata.java index e7f27b875..9ced32eaf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageMetadata.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageMetadata.java @@ -78,7 +78,7 @@ public static ImageMetadata from(ImageMetadata obj) { /** * Sets the region, size, rotation, quality and extension parameters from the IIIF request */ - public void setIIIFParameters(ImageRequest imageRequest){ + public void setIIIFParameters(ImageRequest imageRequest) { this.setRegion(imageRequest.getRegion()); this.setSize(imageRequest.getSize()); this.setRotation(imageRequest.getRotation()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageRequest.java index 73e542e7d..4ad3e5366 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageRequest.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/ImageRequest.java @@ -127,8 +127,8 @@ public void downloadImage(String filePath, String fileName) throws IOException { /** * Downloads and saves the image to the local filesystem * - * @param filePath The path of the directory where the image should be saved - * @param fileName The name that should be given to the saved image + * @param filePath The path of the directory where the image should be saved + * @param fileName The name that should be given to the saved image * @param requestUrl The complete IIIF Image API compliant URL of the image resource. Useful when URL doesn't need to be generated or has to be overridden. * @throws IOException If the image could not downloaded or written to the filesystem */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v2/ImageInformationRequest_v2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v2/ImageInformationRequest_v2.java index 092bd8e38..7ee7eff8e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v2/ImageInformationRequest_v2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v2/ImageInformationRequest_v2.java @@ -57,7 +57,9 @@ public void saveToFile(String filePath, String fileName) throws IOException { LOGGER.debug("Image information request's json response data written to file successfully. Request url:\t" + url); } - /** Get the {@link ImageApiVersion} of the ImageInformation */ + /** + * Get the {@link ImageApiVersion} of the ImageInformation + */ public ImageApiVersion getImageApiVersion() { return new ImageApiVersion(IMAGE_API_VERSION.TWO_POINT_ONE_POINT_ONE); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v3/ImageInformationRequest_v3.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v3/ImageInformationRequest_v3.java index 6e3200e8c..c2c7ede7b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v3/ImageInformationRequest_v3.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/imageapi/v3/ImageInformationRequest_v3.java @@ -57,7 +57,9 @@ public void saveToFile(String filePath, String fileName) throws IOException { LOGGER.debug("Image information request's json response data written to file successfully. Request url:\t" + url); } - /** Get the {@link ImageApiVersion} of the ImageInformation */ + /** + * Get the {@link ImageApiVersion} of the ImageInformation + */ public ImageApiVersion getImageApiVersion() { return new ImageApiVersion(IMAGE_API_VERSION.THREE_POINT_ZERO); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/presentationapi/v2/ManifestFactory.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/presentationapi/v2/ManifestFactory.java index dd5112e20..5873ec355 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/presentationapi/v2/ManifestFactory.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/iiif/presentationapi/v2/ManifestFactory.java @@ -30,7 +30,7 @@ public ManifestFactory(String manifestUrl) throws Exception { * Writes the {@link MetadataJson} generated from this {@link Manifest} to the filesystem * * @param jobDirectoryString The directory where the file has to be written to - * @param filename The name of the file with extension + * @param filename The name of the file with extension */ public void saveMetadataJson(String jobDirectoryString, String filename) { MetadataJson metadataJson = new MetadataJson(manifest); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/Importer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/Importer.java index 763431a02..1c55ef2bd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/Importer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/Importer.java @@ -1,55 +1,54 @@ package org.vitrivr.cineast.core.importer; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; public interface Importer { - /** - * @return the next available element or null in case the end is reached - */ - T readNext(); - - /** - * converts the native type of the importer to a general representation - */ - Map convert(T data); - - /** - * @return the next available element converted to a general representation or null in case the end is reached - */ - default Map readNextAsMap(){ - T next = readNext(); - if(next == null){ - return null; - } - return convert(next); - } - - /** - * @return a list of all remaining elements from the current position to the end - */ - default List readAllRemaining(){ - ArrayList _return = new ArrayList<>(); - - T t = null; - while((t = readNext()) != null){ - _return.add(t); - } - return _return; - } - - default List> readAllRemainingAsMap(){ - ArrayList> _return = new ArrayList<>(); - - Map next = null; - while((next = readNextAsMap()) != null){ - _return.add(next); - } - return _return; - } - + /** + * @return the next available element or null in case the end is reached + */ + T readNext(); + + /** + * converts the native type of the importer to a general representation + */ + Map convert(T data); + + /** + * @return the next available element converted to a general representation or null in case the end is reached + */ + default Map readNextAsMap() { + T next = readNext(); + if (next == null) { + return null; + } + return convert(next); + } + + /** + * @return a list of all remaining elements from the current position to the end + */ + default List readAllRemaining() { + ArrayList _return = new ArrayList<>(); + + T t = null; + while ((t = readNext()) != null) { + _return.add(t); + } + return _return; + } + + default List> readAllRemainingAsMap() { + ArrayList> _return = new ArrayList<>(); + + Map next = null; + while ((next = readNextAsMap()) != null) { + _return.add(next); + } + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/JsonObjectImporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/JsonObjectImporter.java index 682718ea5..2645d9b30 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/JsonObjectImporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/JsonObjectImporter.java @@ -6,15 +6,14 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.providers.primitive.BitSetTypeProvider; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.providers.primitive.BitSetTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; public class JsonObjectImporter implements Importer { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/TupleInsertMessageImporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/TupleInsertMessageImporter.java index 911904527..1a8b43b5c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/TupleInsertMessageImporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/importer/TupleInsertMessageImporter.java @@ -1,5 +1,12 @@ package org.vitrivr.cineast.core.importer; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.adampro.grpc.AdamGrpc.DataMessage; @@ -7,48 +14,44 @@ import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.db.DataMessageConverter; -import java.io.*; -import java.util.HashMap; -import java.util.Map; - public class TupleInsertMessageImporter implements Importer { - private final FileInputStream inStream; - private static final Logger LOGGER = LogManager.getLogger(); - - public TupleInsertMessageImporter(File inputFile) throws FileNotFoundException{ - this.inStream = new FileInputStream(inputFile); - } - - @Override - public TupleInsertMessage readNext(){ - while(true){ - try { - return TupleInsertMessage.parseDelimitedFrom(this.inStream); - } catch (EOFException eof) { - return null; - } catch (IOException e) { - LOGGER.error("error while reading TupleInsertMessage, skipping"); - } - } - } - - @Override - public Map convert(TupleInsertMessage message) { - if(message == null){ - return null; - } - - Map data = message.getDataMap(); - - HashMap map = new HashMap<>(); - - for(String key : data.keySet()){ - map.put(key, DataMessageConverter.convert(data.get(key))); - } - - return map; - } - - + private final FileInputStream inStream; + private static final Logger LOGGER = LogManager.getLogger(); + + public TupleInsertMessageImporter(File inputFile) throws FileNotFoundException { + this.inStream = new FileInputStream(inputFile); + } + + @Override + public TupleInsertMessage readNext() { + while (true) { + try { + return TupleInsertMessage.parseDelimitedFrom(this.inStream); + } catch (EOFException eof) { + return null; + } catch (IOException e) { + LOGGER.error("error while reading TupleInsertMessage, skipping"); + } + } + } + + @Override + public Map convert(TupleInsertMessage message) { + if (message == null) { + return null; + } + + Map data = message.getDataMap(); + + HashMap map = new HashMap<>(); + + for (String key : data.keySet()) { + map.put(key, DataMessageConverter.convert(data.get(key))); + } + + return map; + } + + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java index ded85ae0e..9f0d541ac 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java @@ -1,9 +1,22 @@ package org.vitrivr.cineast.core.render; -import com.jogamp.opengl.*; +import com.jogamp.opengl.GL; +import com.jogamp.opengl.GL2; +import com.jogamp.opengl.GL2ES3; +import com.jogamp.opengl.GL2GL3; +import com.jogamp.opengl.GLCapabilities; +import com.jogamp.opengl.GLContext; +import com.jogamp.opengl.GLDrawableFactory; +import com.jogamp.opengl.GLOffscreenAutoDrawable; +import com.jogamp.opengl.GLProfile; import com.jogamp.opengl.fixedfunc.GLMatrixFunc; import com.jogamp.opengl.glu.GLU; import com.jogamp.opengl.util.awt.AWTGLReadBufferUtil; +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.ReentrantLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joml.Vector3f; @@ -11,431 +24,438 @@ import org.vitrivr.cineast.core.data.m3d.ReadableMesh; import org.vitrivr.cineast.core.data.m3d.VoxelGrid; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.locks.ReentrantLock; - /** - * This class can be used to render 3D models (Meshes or Voxel-models) using the JOGL rendering environment. It - * currently has the following features: - * - * - Rendering of single Mesh or VoxelGrid - * - Free positioning of the camera in terms of either cartesian or polar coordinate - * - Snapshot of the rendered image can be obtained at any time. - * - * The class supports offscreen rendering and can be accessed by multipled Threads. However, the multithreading - * model of JOGL requires a thread to retain() and release() the JOGLOffscreenRenderer before rendering anything - * by calling the respective function. + * This class can be used to render 3D models (Meshes or Voxel-models) using the JOGL rendering environment. It currently has the following features: + *

    + * - Rendering of single Mesh or VoxelGrid - Free positioning of the camera in terms of either cartesian or polar coordinate - Snapshot of the rendered image can be obtained at any time. + *

    + * The class supports offscreen rendering and can be accessed by multipled Threads. However, the multithreading model of JOGL requires a thread to retain() and release() the JOGLOffscreenRenderer before rendering anything by calling the respective function. * * @see Mesh * @see VoxelGrid - * */ public class JOGLOffscreenRenderer implements Renderer { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Default GLProfile to be used. Should be GL2. */ - private static final GLProfile GL_PROFILE = GLProfile.get(GLProfile.GL2); - - /** GLCapabilities. Can be used to enable/disable hardware acceleration etc. */ - private static final GLCapabilities GL_CAPABILITIES = new GLCapabilities(GL_PROFILE); - - /** OpenGL Utility Library reference */ - private final GLU glu; - - /** OpenGL drawable reference. */ - private final GLOffscreenAutoDrawable drawable; - /** OpenGL context reference used for drawing. */ - private final GL2 gl; - - /** Width of the JOGLOffscreenRenderer in pixels. */ - private final int width; - - /** Height of the JOGLOffscreenRenderer in pixels. */ - private final int height; - - /** Aspect-ratio of the JOGLOffscreenRenderer. */ - private final float aspect; - - /** Polygon-mode used during rendering. */ - private int polygonmode = GL2GL3.GL_FILL; - - /** Lock that makes sure that only a single Thread is using the classes rendering facility at a time. */ - private ReentrantLock lock = new ReentrantLock(true); - - /** List of object handles that should be rendered. */ - private final List objects = new ArrayList<>(); - - /* - * This code-block can be used to configure the off-screen renderer's GL_CAPABILITIES. - */ - static { - GL_CAPABILITIES.setOnscreen(false); - GL_CAPABILITIES.setHardwareAccelerated(true); - } - - /** - * Default constructor. Defines the width and the height of this JOGLOffscreenRenderer and - * initializes all the required OpenGL bindings. - * - * @param width Width in pixels. - * @param height Height in pixels. - */ - public JOGLOffscreenRenderer(int width, int height) { - /* Assign width and height. */ - this.width = width; - this.height = height; - this.aspect = (float) width / (float) height; - - /* Initialize GLOffscreenAutoDrawable. */ - GLDrawableFactory factory = GLDrawableFactory.getFactory(GL_PROFILE); - this.drawable = factory.createOffscreenAutoDrawable(null, GL_CAPABILITIES,null,width,height); - this.drawable.display(); - - /* Initialize GLU and GL2. */ - this.glu = new GLU(); - this.gl = drawable.getGL().getGL2(); - - /* Set default color. */ - gl.glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - } - - /** - * Getter for width. - * - * @return Width of the JOGLOffscreenRenderer. - */ - public final int getWidth() { - return width; - } - - /** - * Getter for height. - * - * @return Height of the JOGLOffscreenRenderer. - */ - public final int getHeight() { - return height; - } - - /** - * Getter for aspect. - * - * @return Aspect ratio of the JOGLOffscreenRenderer. - */ - public final float getAspect() { - return aspect; + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Default GLProfile to be used. Should be GL2. + */ + private static final GLProfile GL_PROFILE = GLProfile.get(GLProfile.GL2); + + /** + * GLCapabilities. Can be used to enable/disable hardware acceleration etc. + */ + private static final GLCapabilities GL_CAPABILITIES = new GLCapabilities(GL_PROFILE); + + /** + * OpenGL Utility Library reference + */ + private final GLU glu; + + /** + * OpenGL drawable reference. + */ + private final GLOffscreenAutoDrawable drawable; + + /** + * OpenGL context reference used for drawing. + */ + private final GL2 gl; + + /** + * Width of the JOGLOffscreenRenderer in pixels. + */ + private final int width; + + /** + * Height of the JOGLOffscreenRenderer in pixels. + */ + private final int height; + + /** + * Aspect-ratio of the JOGLOffscreenRenderer. + */ + private final float aspect; + + /** + * Polygon-mode used during rendering. + */ + private int polygonmode = GL2GL3.GL_FILL; + + /** + * Lock that makes sure that only a single Thread is using the classes rendering facility at a time. + */ + private ReentrantLock lock = new ReentrantLock(true); + + /** + * List of object handles that should be rendered. + */ + private final List objects = new ArrayList<>(); + + /* + * This code-block can be used to configure the off-screen renderer's GL_CAPABILITIES. + */ + static { + GL_CAPABILITIES.setOnscreen(false); + GL_CAPABILITIES.setHardwareAccelerated(true); + } + + /** + * Default constructor. Defines the width and the height of this JOGLOffscreenRenderer and initializes all the required OpenGL bindings. + * + * @param width Width in pixels. + * @param height Height in pixels. + */ + public JOGLOffscreenRenderer(int width, int height) { + /* Assign width and height. */ + this.width = width; + this.height = height; + this.aspect = (float) width / (float) height; + + /* Initialize GLOffscreenAutoDrawable. */ + GLDrawableFactory factory = GLDrawableFactory.getFactory(GL_PROFILE); + this.drawable = factory.createOffscreenAutoDrawable(null, GL_CAPABILITIES, null, width, height); + this.drawable.display(); + + /* Initialize GLU and GL2. */ + this.glu = new GLU(); + this.gl = drawable.getGL().getGL2(); + + /* Set default color. */ + gl.glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + } + + /** + * Getter for width. + * + * @return Width of the JOGLOffscreenRenderer. + */ + public final int getWidth() { + return width; + } + + /** + * Getter for height. + * + * @return Height of the JOGLOffscreenRenderer. + */ + public final int getHeight() { + return height; + } + + /** + * Getter for aspect. + * + * @return Aspect ratio of the JOGLOffscreenRenderer. + */ + public final float getAspect() { + return aspect; + } + + /** + * Getter for polygonmode. + * + * @return Polygonmode for drawing, either GL_POINT, GL_LINE or GL_FILL. + */ + public int getPolygonmode() { + return polygonmode; + } + + /** + * Setter for polygonmode. + * + * @param polygonmode Polygonmode for drawing, either GL_POINT, GL_LINE or GL_FILL. + */ + public synchronized void setPolygonmode(int polygonmode) { + if (polygonmode == GL2GL3.GL_POINT || polygonmode == GL2GL3.GL_LINE || polygonmode == GL2GL3.GL_FILL) { + this.polygonmode = polygonmode; } - - /** - * Getter for polygonmode. - * - * @return Polygonmode for drawing, either GL_POINT, GL_LINE or GL_FILL. - */ - public int getPolygonmode() { - return polygonmode; + } + + /** + * + */ + @Override + public void render() { + /* Clear context. */ + gl.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT); + + /* Switch matrix mode to modelview. */ + gl.glMatrixMode(GLMatrixFunc.GL_MODELVIEW); + gl.glEnable(GL.GL_DEPTH_TEST); + gl.glDepthFunc(GL.GL_LESS); + gl.glLoadIdentity(); + gl.glPolygonMode(GL.GL_FRONT_AND_BACK, this.polygonmode); + + /* Call list. */ + for (Integer handle : this.objects) { + gl.glCallList(handle); } - - /** - * Setter for polygonmode. - * - * @param polygonmode Polygonmode for drawing, either GL_POINT, GL_LINE or GL_FILL. - */ - public synchronized void setPolygonmode(int polygonmode) { - if (polygonmode == GL2GL3.GL_POINT || polygonmode == GL2GL3.GL_LINE || polygonmode == GL2GL3.GL_FILL) { - this.polygonmode = polygonmode; - } - } - - /** - * - */ - @Override - public void render() { - /* Clear context. */ - gl.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT); - - /* Switch matrix mode to modelview. */ - gl.glMatrixMode(GLMatrixFunc.GL_MODELVIEW); - gl.glEnable(GL.GL_DEPTH_TEST); - gl.glDepthFunc(GL.GL_LESS); - gl.glLoadIdentity(); - gl.glPolygonMode(GL.GL_FRONT_AND_BACK, this.polygonmode); - - /* Call list. */ - for (Integer handle : this.objects) { - gl.glCallList(handle); + } + + /** + * Renders a new Mesh object and thereby removes any previously rendered object + * + * @param mesh Mesh that should be rendered + */ + @Override + public void assemble(ReadableMesh mesh) { + int meshList = gl.glGenLists(1); + this.objects.add(meshList); + gl.glNewList(meshList, GL2.GL_COMPILE); + { + for (Mesh.Face face : mesh.getFaces()) { + /* Extract normals and vertices. */ + List vertices = face.getVertices(); + + /* Determine gl_draw_type. */ + int gl_draw_type = GL.GL_TRIANGLES; + if (face.getType() == Mesh.FaceType.QUAD) { + gl_draw_type = GL2ES3.GL_QUADS; } - } - /** - * Renders a new Mesh object and thereby removes any previously rendered object - * - * @param mesh Mesh that should be rendered - */ - @Override - public void assemble(ReadableMesh mesh) { - int meshList = gl.glGenLists(1); - this.objects.add(meshList); - gl.glNewList(meshList, GL2.GL_COMPILE); + /* Drawing is handled differently depending on whether its a TRI or QUAD mesh. */ + gl.glBegin(gl_draw_type); { - for (Mesh.Face face : mesh.getFaces()) { - /* Extract normals and vertices. */ - List vertices = face.getVertices(); - - /* Determine gl_draw_type. */ - int gl_draw_type = GL.GL_TRIANGLES; - if (face.getType() == Mesh.FaceType.QUAD) { - gl_draw_type = GL2ES3.GL_QUADS; - } - - /* Drawing is handled differently depending on whether its a TRI or QUAD mesh. */ - gl.glBegin(gl_draw_type); - { - for (Mesh.Vertex vertex : vertices) { - gl.glColor3f(vertex.getColor().x(), vertex.getColor().y(), vertex.getColor().z()); - gl.glVertex3f(vertex.getPosition().x(), vertex.getPosition().y(), vertex.getPosition().z()); - gl.glNormal3f(vertex.getNormal().x(), vertex.getNormal().y(), vertex.getNormal().z()); - } - } - gl.glEnd(); - } + for (Mesh.Vertex vertex : vertices) { + gl.glColor3f(vertex.getColor().x(), vertex.getColor().y(), vertex.getColor().z()); + gl.glVertex3f(vertex.getPosition().x(), vertex.getPosition().y(), vertex.getPosition().z()); + gl.glNormal3f(vertex.getNormal().x(), vertex.getNormal().y(), vertex.getNormal().z()); + } } - gl.glEndList(); + gl.glEnd(); + } } - - /** - * Assembles a new VoxelGrid object and thereby adds it to the list of objects that - * should be rendered. - * - * @param grid VoxelGrid that should be rendered. - */ - @Override - public void assemble(VoxelGrid grid) { - int meshList = gl.glGenLists(1); - this.objects.add(meshList); - gl.glNewList(meshList, GL2.GL_COMPILE); - { - boolean[] visible = {true, true, true, true, true, true}; - - for (int i = 0; i < grid.getSizeX(); i++) { - for (int j = 0; j < grid.getSizeY(); j++) { - for (int k = 0; k < grid.getSizeZ(); k++) { - /* Skip Voxel if its inactive. */ - if (grid.get(i,j,k) == VoxelGrid.Voxel.INVISIBLE) { - continue; - } - - Vector3f voxelCenter = grid.getVoxelCenter(i,j,k); - - /* Extract center of the voxel. */ - float x = voxelCenter.x; - float y = voxelCenter.y; - float z = voxelCenter.z; - - /* Determine which faces to draw: Faced that are covered by another active voxel are switched off. */ - if(i > 0) { - visible[0] = !grid.isVisible(i-1,j,k); - } - if(i < grid.getSizeX()-1) { - visible[1] = !grid.isVisible(i+1,j,k); - } - if(j > 0) { - visible[2] = !grid.isVisible(i,j-1,k); - } - if(j < grid.getSizeY()-1) { - visible[3] = !grid.isVisible(i,j+1,k); - } - if(k > 0) { - visible[4] = !grid.isVisible(i,j,k-1); - } - if(k < grid.getSizeZ()-1) { - visible[5] = !grid.isVisible(i,j,k+1); - } - - /* Draw the cube. */ - gl.glBegin(GL2ES3.GL_QUADS); - { - final float halfresolution = grid.getResolution()/2.0f; - - /* 1 */ - if (visible[0]) { - gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); - gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); - } - - /* 2 */ - if (visible[1]) { - gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); - gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); - gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); - } - - /* 3 */ - if (visible[2]) { - gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); - gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); - gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); - } - - /* 4 */ - if (visible[3]) { - gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); - } - - /* 5 */ - if (visible[4]) { - gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); - gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); - gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); - } - - /* 6 */ - if (visible[5]) { - gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); - gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); - gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); - } - } - gl.glEnd(); - } - } + gl.glEndList(); + } + + /** + * Assembles a new VoxelGrid object and thereby adds it to the list of objects that should be rendered. + * + * @param grid VoxelGrid that should be rendered. + */ + @Override + public void assemble(VoxelGrid grid) { + int meshList = gl.glGenLists(1); + this.objects.add(meshList); + gl.glNewList(meshList, GL2.GL_COMPILE); + { + boolean[] visible = {true, true, true, true, true, true}; + + for (int i = 0; i < grid.getSizeX(); i++) { + for (int j = 0; j < grid.getSizeY(); j++) { + for (int k = 0; k < grid.getSizeZ(); k++) { + /* Skip Voxel if its inactive. */ + if (grid.get(i, j, k) == VoxelGrid.Voxel.INVISIBLE) { + continue; } - } - gl.glEndList(); - } - /** - * Changes the positionCamera of the camera. - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - * @param cx x Position of the object of interest (i.e. the point at which the camera looks). - * @param cy y Position of the object of interest (i.e. the point at which the camera looks). - * @param cz z Position of the object of interest (i.e. the point at which the camera looks). - * @param upx x-direction of the camera's UP position. - * @param upy y-direction of the camera's UP position. - * @param upz z-direction of the camera's UP position. - */ - @Override - public final void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz, double upx, double upy, double upz) { - /* Check context. */ - if (!this.checkContext()) { - return; - } + Vector3f voxelCenter = grid.getVoxelCenter(i, j, k); - /* Switch matrix mode to projection. */ - gl.glMatrixMode(GLMatrixFunc.GL_PROJECTION); - gl.glLoadIdentity(); + /* Extract center of the voxel. */ + float x = voxelCenter.x; + float y = voxelCenter.y; + float z = voxelCenter.z; - /* Set default perspective. */ - glu.gluPerspective(45.0f, this.aspect, 0.01f, 100.0f); + /* Determine which faces to draw: Faced that are covered by another active voxel are switched off. */ + if (i > 0) { + visible[0] = !grid.isVisible(i - 1, j, k); + } + if (i < grid.getSizeX() - 1) { + visible[1] = !grid.isVisible(i + 1, j, k); + } + if (j > 0) { + visible[2] = !grid.isVisible(i, j - 1, k); + } + if (j < grid.getSizeY() - 1) { + visible[3] = !grid.isVisible(i, j + 1, k); + } + if (k > 0) { + visible[4] = !grid.isVisible(i, j, k - 1); + } + if (k < grid.getSizeZ() - 1) { + visible[5] = !grid.isVisible(i, j, k + 1); + } - /* Update camera position. */ - glu.gluLookAt(ex,ey,ez,cx,cy,cz,upx,upy,upz); + /* Draw the cube. */ + gl.glBegin(GL2ES3.GL_QUADS); + { + final float halfresolution = grid.getResolution() / 2.0f; + + /* 1 */ + if (visible[0]) { + gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); + gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); + } + + /* 2 */ + if (visible[1]) { + gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); + gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); + gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); + } + + /* 3 */ + if (visible[2]) { + gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); + gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); + gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); + } + + /* 4 */ + if (visible[3]) { + gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); + } + + /* 5 */ + if (visible[4]) { + gl.glVertex3f(x - halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x + halfresolution, y - halfresolution, z - halfresolution); + gl.glVertex3f(x + halfresolution, y - halfresolution, z + halfresolution); + gl.glVertex3f(x - halfresolution, y - halfresolution, z + halfresolution); + } + + /* 6 */ + if (visible[5]) { + gl.glVertex3f(x + halfresolution, y + halfresolution, z - halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z - halfresolution); + gl.glVertex3f(x - halfresolution, y + halfresolution, z + halfresolution); + gl.glVertex3f(x + halfresolution, y + halfresolution, z + halfresolution); + } + } + gl.glEnd(); + } + } + } } - - /** - * Clears buffers to preset-values. - */ - @Override - public final void clear() { - this.clear(Color.BLACK); + gl.glEndList(); + } + + /** + * Changes the positionCamera of the camera. + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + * @param cx x Position of the object of interest (i.e. the point at which the camera looks). + * @param cy y Position of the object of interest (i.e. the point at which the camera looks). + * @param cz z Position of the object of interest (i.e. the point at which the camera looks). + * @param upx x-direction of the camera's UP position. + * @param upy y-direction of the camera's UP position. + * @param upz z-direction of the camera's UP position. + */ + @Override + public final void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz, double upx, double upy, double upz) { + /* Check context. */ + if (!this.checkContext()) { + return; } - /** - * Clears buffers to preset-values and applies a user-defined background colour. - * - * @param color The background colour to be used. - */ - @Override - public void clear(Color color) { - if (!this.checkContext()) { - return; - } - for (Integer handle : this.objects) { - gl.glDeleteLists(handle, 1); - } - gl.glClearColorIi(color.getRed(), color.getGreen(), color.getBlue(), color.getAlpha()); - gl.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT); - this.objects.clear(); + /* Switch matrix mode to projection. */ + gl.glMatrixMode(GLMatrixFunc.GL_PROJECTION); + gl.glLoadIdentity(); + + /* Set default perspective. */ + glu.gluPerspective(45.0f, this.aspect, 0.01f, 100.0f); + + /* Update camera position. */ + glu.gluLookAt(ex, ey, ez, cx, cy, cz, upx, upy, upz); + } + + /** + * Clears buffers to preset-values. + */ + @Override + public final void clear() { + this.clear(Color.BLACK); + } + + /** + * Clears buffers to preset-values and applies a user-defined background colour. + * + * @param color The background colour to be used. + */ + @Override + public void clear(Color color) { + if (!this.checkContext()) { + return; } - - /** - * Obtains and returns a BufferedImage in AWT orientation from the current JOGLOffscreenRenderer. - * - * @return BufferedImage containing a snapshot of the current render-buffer. - */ - @Override - public final BufferedImage obtain() { - /* Create and return a BufferedImage from buffer. */ - if (!this.checkContext()) { - return null; - } - AWTGLReadBufferUtil glReadBufferUtil = new AWTGLReadBufferUtil(gl.getGL2().getGLProfile(), false); - return glReadBufferUtil.readPixelsToBufferedImage(gl.getGL2(), true); + for (Integer handle : this.objects) { + gl.glDeleteLists(handle, 1); } - - /** - * Makes the current thread try to retain the GLContext of this JOGLOffscreenRenderer. The - * method returns true upon success and false otherwise. - * - * Important: Only one thread can retain a GLContext at a time. Relinquish the thread by - * calling release(). - * - * @return True if GLContext was retained and false otherwise. - */ - @Override - public final boolean retain() { - this.lock.lock(); - int result = this.gl.getContext().makeCurrent(); - if (result == GLContext.CONTEXT_CURRENT_NEW || result == GLContext.CONTEXT_CURRENT) { - return true; - } else { - this.lock.unlock(); - LOGGER.error("Thread '{}' failed to retain JOGLOffscreenRenderer.", Thread.currentThread().getName()); - return false; - } - + gl.glClearColorIi(color.getRed(), color.getGreen(), color.getBlue(), color.getAlpha()); + gl.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT); + this.objects.clear(); + } + + /** + * Obtains and returns a BufferedImage in AWT orientation from the current JOGLOffscreenRenderer. + * + * @return BufferedImage containing a snapshot of the current render-buffer. + */ + @Override + public final BufferedImage obtain() { + /* Create and return a BufferedImage from buffer. */ + if (!this.checkContext()) { + return null; } - - /** - * Makes the current thread release its ownership of the current JOGLOffscreenRenderer's GLContext. - */ - @Override - public final void release() { - if (this.checkContext()) { - this.gl.getContext().release(); - this.lock.unlock(); - } + AWTGLReadBufferUtil glReadBufferUtil = new AWTGLReadBufferUtil(gl.getGL2().getGLProfile(), false); + return glReadBufferUtil.readPixelsToBufferedImage(gl.getGL2(), true); + } + + /** + * Makes the current thread try to retain the GLContext of this JOGLOffscreenRenderer. The method returns true upon success and false otherwise. + * + * Important: Only one thread can retain a GLContext at a time. Relinquish the thread by + * calling release(). + * + * @return True if GLContext was retained and false otherwise. + */ + @Override + public final boolean retain() { + this.lock.lock(); + int result = this.gl.getContext().makeCurrent(); + if (result == GLContext.CONTEXT_CURRENT_NEW || result == GLContext.CONTEXT_CURRENT) { + return true; + } else { + this.lock.unlock(); + LOGGER.error("Thread '{}' failed to retain JOGLOffscreenRenderer.", Thread.currentThread().getName()); + return false; } + } - /** - * Checks if the thread the GLContext is assigned to is equal to the Thread the current - * code is being executed in. - * - * @return True if context-thread is equal to current thread and false otherwise, - */ - private boolean checkContext() { - if (!this.lock.isHeldByCurrentThread()) { - LOGGER.error("Cannot access JOGLOffscreenRenderer because current thread '{}' does not own its GLContext.", Thread.currentThread().getName()); - return false; - } else { - return true; - } + /** + * Makes the current thread release its ownership of the current JOGLOffscreenRenderer's GLContext. + */ + @Override + public final void release() { + if (this.checkContext()) { + this.gl.getContext().release(); + this.lock.unlock(); + } + } + + + /** + * Checks if the thread the GLContext is assigned to is equal to the Thread the current code is being executed in. + * + * @return True if context-thread is equal to current thread and false otherwise, + */ + private boolean checkContext() { + if (!this.lock.isHeldByCurrentThread()) { + LOGGER.error("Cannot access JOGLOffscreenRenderer because current thread '{}' does not own its GLContext.", Thread.currentThread().getName()); + return false; + } else { + return true; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java index 8fb6ff11f..495f3afd3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java @@ -1,195 +1,182 @@ package org.vitrivr.cineast.core.render; +import java.awt.Color; +import java.awt.image.BufferedImage; import org.vitrivr.cineast.core.data.m3d.ReadableMesh; import org.vitrivr.cineast.core.data.m3d.VoxelGrid; -import java.awt.*; -import java.awt.image.BufferedImage; - /** - * This interface defines methods that a renderer for 3D models (e.g. Meshes or Voxels must implement). It - * currently provides the following features: - * - * - Rendering of single Mesh or VoxelGrid - * - Free positioning of the camera in terms of either cartesian or polar coordinate - * - Snapshot of the rendered image can be obtained at any time. - * + * This interface defines methods that a renderer for 3D models (e.g. Meshes or Voxels must implement). It currently provides the following features: + *

    + * - Rendering of single Mesh or VoxelGrid - Free positioning of the camera in terms of either cartesian or polar coordinate - Snapshot of the rendered image can be obtained at any time. */ public interface Renderer { - /** - * Renders the scene of the renderable using all the configured objects - * and the configured camera position. - */ - void render(); - - /** - * Assembles a new Mesh object and thereby adds it to the list of objects that - * should be rendered. - * - * @param mesh Mesh that should be rendered - */ - void assemble(ReadableMesh mesh); - - /** - * Assembles a new VoxelGrid object and thereby adds it to the list of objects that - * should be rendered. - * - * @param grid VoxelGrid that should be rendered. - */ - void assemble(VoxelGrid grid); - - /** - * Changes the positionCamera of the camera. - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - * @param cx x Position of the object of interest (i.e. the point at which the camera looks). - * @param cy y Position of the object of interest (i.e. the point at which the camera looks). - * @param cz z Position of the object of interest (i.e. the point at which the camera looks). - * @param upx x-direction of the camera's UP position. - * @param upy y-direction of the camera's UP position. - * @param upz z-direction of the camera's UP position. - */ - void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz, double upx, double upy, double upz); - - /** - * Changes the positionCamera of the camera. - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - * @param cx x Position of the object of interest (i.e. the point at which the camera looks). - * @param cy y Position of the object of interest (i.e. the point at which the camera looks). - * @param cz z Position of the object of interest (i.e. the point at which the camera looks). - */ - default void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz) { - positionCamera(ex, ey, ez, cx,cy,cz,0.0,1.0,0.0); - } - - /** - * Changes the positionCamera of the camera. The camera can be freely rotated around the origin [1,0,0] (cartesian - * coordinates) and it can take any distance from that same origin. - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - * @param cx x Position of the object of interest (i.e. the point at which the camera looks). - * @param cy y Position of the object of interest (i.e. the point at which the camera looks). - * @param cz z Position of the object of interest (i.e. the point at which the camera looks). - */ - default void positionCamera(float ex, float ey, float ez, float cx, float cy, float cz) { - this.positionCamera((double)ex,(double)ey,(double)ez,(double)cx,(double)cy,(double)cz); - } - - /** - * Changes the positionCamera of the camera. This method should make sure, that camera always points - * towards the origin [0,0,0] - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - */ - default void positionCamera(double ex, double ey, double ez) { - this.positionCamera(ex, ey, ez, 0.0,0.0,0.0); - } - - /** - * Changes the positionCamera of the camera. This method should make sure, that camera always points - * towards the origin [0,0,0] - * - * @param ex x Position of the Camera - * @param ey y Position of the Camera - * @param ez z Position of the Camera - */ - default void positionCamera(float ex, float ey, float ez) { - this.positionCamera(ex, ey, ez, 0.0f,0.0f,0.0f); - } - - /** - * Changes the positionCamera of the camera. The camera can be freely rotated around the origin [0,0,0] - * (cartesian coordinates) and it can take any distance from that same origin (Arc Ball camera). - * - * @param r Distance of the camera from the origin at [0,0,0] - * @param theta Polar angle of the camera (i.e. angle between vector and z-axis) in degree - * @param phi z Azimut angle of the camera (i.e. angle between vector and x-axis) in degree - * @param cx x Position of the object of interest (i.e. the point at which the camera looks). - * @param cy y Position of the object of interest (i.e. the point at which the camera looks). - * @param cz z Position of the object of interest (i.e. the point at which the camera looks). - */ - default void positionCameraPolar(double r, double theta, double phi, double cx, double cy, double cz) { - double theta_rad = Math.toRadians(theta); - double phi_rad = Math.toRadians(phi); - - double x = r * Math.cos(theta_rad) * Math.cos(phi_rad); - double y = r * Math.sin(theta_rad); - double z = r * Math.cos(theta_rad) * Math.sin(phi_rad); - - /* Calculate the RIGHT and the UP vector. */ - double[] look = {x-cx,y-cy,z-cz}; - double[] right = { - look[1] * 0.0f - look[2] * 1.0f, - look[2] * 0.0f - look[0] * 0.0f, - look[0] * 1.0f - look[1] * 0.0f - }; - double[] up = { - look[1] * right[2] - look[2] * right[1], - look[2] * right[0] - look[0] * right[2], - look[0] * right[1] - look[1] * right[0] - }; - - /* Normalize the UP vector. */ - double abs = Math.sqrt(Math.pow(up[0], 2) + Math.pow(up[1], 2) + Math.pow(up[2], 2)); - up[0] /= abs; - up[1] /= abs; - up[2] /= abs; - - /* Re-position the camera. */ - positionCamera((float)x,(float)y,(float)z,cx,cy,cz,up[0],up[1],up[2]); - } - - /** - * Changes the positionCamera of the camera. The camera can be freely rotated around the origin - * [0,0,0] (cartesian coordinates) and it can take any distance from that same origin. - * - * @param r Distance of the camera from (0,0,0) - * @param theta Polar angle of the camera (i.e. angle between vector and z-axis) in degree - * @param phi z Azimut angle of the camera (i.e. angle between vector and x-axis) in degree - * @param cx x Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. - * @param cy y Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. - * @param cz z Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. - */ - default void positionCameraPolar(float r, float theta, float phi, float cx, float cy, float cz) { - this.positionCamera((double)r,(double)theta,(double)phi,(double)cx,(double)cy,(double)cz); - } - - /** - * Obtains and returns a BufferedImage from the current Renderer. - * - * @return BufferedImage containing a snapshot of the current render-buffer. - */ - BufferedImage obtain(); - - /** - * Clears buffers to preset-values and applies a user-defined background colour. - * - * @param color The background colour to be used. - */ - void clear(Color color); - - /** - * Clears buffers to preset-values. - */ - void clear(); - - /** - * Retains control of the Renderer. While a Thread retains a renderer, no other thread should - * be allowed to use it! - */ - boolean retain(); - - /** - * Releases control of the Renderer, making it usable by other Threads again. - */ - void release(); + + /** + * Renders the scene of the renderable using all the configured objects and the configured camera position. + */ + void render(); + + /** + * Assembles a new Mesh object and thereby adds it to the list of objects that should be rendered. + * + * @param mesh Mesh that should be rendered + */ + void assemble(ReadableMesh mesh); + + /** + * Assembles a new VoxelGrid object and thereby adds it to the list of objects that should be rendered. + * + * @param grid VoxelGrid that should be rendered. + */ + void assemble(VoxelGrid grid); + + /** + * Changes the positionCamera of the camera. + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + * @param cx x Position of the object of interest (i.e. the point at which the camera looks). + * @param cy y Position of the object of interest (i.e. the point at which the camera looks). + * @param cz z Position of the object of interest (i.e. the point at which the camera looks). + * @param upx x-direction of the camera's UP position. + * @param upy y-direction of the camera's UP position. + * @param upz z-direction of the camera's UP position. + */ + void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz, double upx, double upy, double upz); + + /** + * Changes the positionCamera of the camera. + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + * @param cx x Position of the object of interest (i.e. the point at which the camera looks). + * @param cy y Position of the object of interest (i.e. the point at which the camera looks). + * @param cz z Position of the object of interest (i.e. the point at which the camera looks). + */ + default void positionCamera(double ex, double ey, double ez, double cx, double cy, double cz) { + positionCamera(ex, ey, ez, cx, cy, cz, 0.0, 1.0, 0.0); + } + + /** + * Changes the positionCamera of the camera. The camera can be freely rotated around the origin [1,0,0] (cartesian coordinates) and it can take any distance from that same origin. + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + * @param cx x Position of the object of interest (i.e. the point at which the camera looks). + * @param cy y Position of the object of interest (i.e. the point at which the camera looks). + * @param cz z Position of the object of interest (i.e. the point at which the camera looks). + */ + default void positionCamera(float ex, float ey, float ez, float cx, float cy, float cz) { + this.positionCamera((double) ex, (double) ey, (double) ez, (double) cx, (double) cy, (double) cz); + } + + /** + * Changes the positionCamera of the camera. This method should make sure, that camera always points towards the origin [0,0,0] + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + */ + default void positionCamera(double ex, double ey, double ez) { + this.positionCamera(ex, ey, ez, 0.0, 0.0, 0.0); + } + + /** + * Changes the positionCamera of the camera. This method should make sure, that camera always points towards the origin [0,0,0] + * + * @param ex x Position of the Camera + * @param ey y Position of the Camera + * @param ez z Position of the Camera + */ + default void positionCamera(float ex, float ey, float ez) { + this.positionCamera(ex, ey, ez, 0.0f, 0.0f, 0.0f); + } + + /** + * Changes the positionCamera of the camera. The camera can be freely rotated around the origin [0,0,0] (cartesian coordinates) and it can take any distance from that same origin (Arc Ball camera). + * + * @param r Distance of the camera from the origin at [0,0,0] + * @param theta Polar angle of the camera (i.e. angle between vector and z-axis) in degree + * @param phi z Azimut angle of the camera (i.e. angle between vector and x-axis) in degree + * @param cx x Position of the object of interest (i.e. the point at which the camera looks). + * @param cy y Position of the object of interest (i.e. the point at which the camera looks). + * @param cz z Position of the object of interest (i.e. the point at which the camera looks). + */ + default void positionCameraPolar(double r, double theta, double phi, double cx, double cy, double cz) { + double theta_rad = Math.toRadians(theta); + double phi_rad = Math.toRadians(phi); + + double x = r * Math.cos(theta_rad) * Math.cos(phi_rad); + double y = r * Math.sin(theta_rad); + double z = r * Math.cos(theta_rad) * Math.sin(phi_rad); + + /* Calculate the RIGHT and the UP vector. */ + double[] look = {x - cx, y - cy, z - cz}; + double[] right = { + look[1] * 0.0f - look[2] * 1.0f, + look[2] * 0.0f - look[0] * 0.0f, + look[0] * 1.0f - look[1] * 0.0f + }; + double[] up = { + look[1] * right[2] - look[2] * right[1], + look[2] * right[0] - look[0] * right[2], + look[0] * right[1] - look[1] * right[0] + }; + + /* Normalize the UP vector. */ + double abs = Math.sqrt(Math.pow(up[0], 2) + Math.pow(up[1], 2) + Math.pow(up[2], 2)); + up[0] /= abs; + up[1] /= abs; + up[2] /= abs; + + /* Re-position the camera. */ + positionCamera((float) x, (float) y, (float) z, cx, cy, cz, up[0], up[1], up[2]); + } + + /** + * Changes the positionCamera of the camera. The camera can be freely rotated around the origin [0,0,0] (cartesian coordinates) and it can take any distance from that same origin. + * + * @param r Distance of the camera from (0,0,0) + * @param theta Polar angle of the camera (i.e. angle between vector and z-axis) in degree + * @param phi z Azimut angle of the camera (i.e. angle between vector and x-axis) in degree + * @param cx x Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. + * @param cy y Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. + * @param cz z Position of the object of interest (i.e. the point at which the camera looks) in cartesian coordinates. + */ + default void positionCameraPolar(float r, float theta, float phi, float cx, float cy, float cz) { + this.positionCamera((double) r, (double) theta, (double) phi, (double) cx, (double) cy, (double) cz); + } + + /** + * Obtains and returns a BufferedImage from the current Renderer. + * + * @return BufferedImage containing a snapshot of the current render-buffer. + */ + BufferedImage obtain(); + + /** + * Clears buffers to preset-values and applies a user-defined background colour. + * + * @param color The background colour to be used. + */ + void clear(Color color); + + /** + * Clears buffers to preset-values. + */ + void clear(); + + /** + * Retains control of the Renderer. While a Thread retains a renderer, no other thread should be allowed to use it! + */ + boolean retain(); + + /** + * Releases control of the Renderer, making it usable by other Threads again. + */ + void release(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ARPartioner.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ARPartioner.java index 1d2d64a3b..45d894a25 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ARPartioner.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ARPartioner.java @@ -1,102 +1,101 @@ package org.vitrivr.cineast.core.util; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.ReadableLabContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.Pair; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; public final class ARPartioner { - private ARPartioner(){} - - public static ArrayList> partition(List input, int width, int height, int angularSegments, int radialSegments){ - ArrayList> _return = new ArrayList<>(angularSegments * radialSegments); - for(int i = 0; i < angularSegments * radialSegments; ++i){ - _return.add(new LinkedList()); - } - - float centerX = width / 2f, centerY = height / 2f, w = width, h = height; - - for(int i = 0; i < input.size(); ++i){ - float x = ((i % width) - centerX) / w, - y = ((i / width) - centerY) / h; - - - //to polar - float r = (float) Math.sqrt(x * x + y * y); - float phi = (float) (Math.atan2(y, x) + Math.PI); - - int radialSegment = r >= 0.5f ? radialSegments - 1 : (int) Math.floor(r * 2 * radialSegments); - int angularSegment = ((int) Math.floor((phi / (2 * Math.PI)) * angularSegments)) % angularSegments; - - int index = radialSegment + radialSegments * angularSegment; - - _return.get(index).add(input.get(i)); - - } - - return _return; - } - - public static Pair partitionImage(MultiImage img, int angularSegments, int radialSegments){ - int[] colors = img.getColors(); - ArrayList tmpList = new ArrayList<>(colors.length); - for(int c : colors){ - tmpList.add(c); - } - - ArrayList> partitions = ARPartioner.partition(tmpList, img.getWidth(), img.getHeight(), angularSegments, radialSegments); - ArrayList stats = new ArrayList(angularSegments * radialSegments * 3); - ArrayList alphas = new ArrayList(angularSegments * radialSegments); - - for(int i = 0; i < angularSegments * radialSegments * 3; ++i){ - stats.add(new SummaryStatistics()); - } - - for(int i = 0; i < angularSegments * radialSegments; ++i){ - alphas.add(new SummaryStatistics()); - } - - for(int i = 0; i < angularSegments * radialSegments; ++i){ - LinkedList cols = partitions.get(i); - SummaryStatistics L = stats.get(3 * i); - SummaryStatistics a = stats.get(3 * i + 1); - SummaryStatistics b = stats.get(3 * i + 2); - SummaryStatistics alpha = alphas.get(i); - - for(int c : cols){ - ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(c); - L.addValue(lab.getL()); - a.addValue(lab.getA()); - b.addValue(lab.getB()); - alpha.addValue(ReadableRGBContainer.getAlpha(c) / 255f); - } - } - - float[] vec = new float[stats.size() * 2]; - float[] weights = new float[vec.length]; - - for(int i = 0; i < stats.size(); ++i){ - SummaryStatistics s = stats.get(i); - vec[2 * i] = (float) s.getMean(); - vec[2 * i + 1] = (float) s.getVariance(); - } - - for(int i = 0; i < alphas.size(); ++i){ - weights[3 * i] = (float) alphas.get(i).getMean(); - weights[3 * i + 1] = weights[3 * i]; - weights[3 * i + 2] = weights[3 * i]; - } - - return new Pair(new FloatVectorImpl(vec), weights); - } - + private ARPartioner() { + } + + public static ArrayList> partition(List input, int width, int height, int angularSegments, int radialSegments) { + ArrayList> _return = new ArrayList<>(angularSegments * radialSegments); + for (int i = 0; i < angularSegments * radialSegments; ++i) { + _return.add(new LinkedList()); + } + + float centerX = width / 2f, centerY = height / 2f, w = width, h = height; + + for (int i = 0; i < input.size(); ++i) { + float x = ((i % width) - centerX) / w, + y = ((i / width) - centerY) / h; + + //to polar + float r = (float) Math.sqrt(x * x + y * y); + float phi = (float) (Math.atan2(y, x) + Math.PI); + + int radialSegment = r >= 0.5f ? radialSegments - 1 : (int) Math.floor(r * 2 * radialSegments); + int angularSegment = ((int) Math.floor((phi / (2 * Math.PI)) * angularSegments)) % angularSegments; + + int index = radialSegment + radialSegments * angularSegment; + + _return.get(index).add(input.get(i)); + + } + + return _return; + } + + public static Pair partitionImage(MultiImage img, int angularSegments, int radialSegments) { + int[] colors = img.getColors(); + ArrayList tmpList = new ArrayList<>(colors.length); + for (int c : colors) { + tmpList.add(c); + } + + ArrayList> partitions = ARPartioner.partition(tmpList, img.getWidth(), img.getHeight(), angularSegments, radialSegments); + ArrayList stats = new ArrayList(angularSegments * radialSegments * 3); + ArrayList alphas = new ArrayList(angularSegments * radialSegments); + + for (int i = 0; i < angularSegments * radialSegments * 3; ++i) { + stats.add(new SummaryStatistics()); + } + + for (int i = 0; i < angularSegments * radialSegments; ++i) { + alphas.add(new SummaryStatistics()); + } + + for (int i = 0; i < angularSegments * radialSegments; ++i) { + LinkedList cols = partitions.get(i); + SummaryStatistics L = stats.get(3 * i); + SummaryStatistics a = stats.get(3 * i + 1); + SummaryStatistics b = stats.get(3 * i + 2); + SummaryStatistics alpha = alphas.get(i); + + for (int c : cols) { + ReadableLabContainer lab = ColorConverter.cachedRGBtoLab(c); + L.addValue(lab.getL()); + a.addValue(lab.getA()); + b.addValue(lab.getB()); + alpha.addValue(ReadableRGBContainer.getAlpha(c) / 255f); + } + } + + float[] vec = new float[stats.size() * 2]; + float[] weights = new float[vec.length]; + + for (int i = 0; i < stats.size(); ++i) { + SummaryStatistics s = stats.get(i); + vec[2 * i] = (float) s.getMean(); + vec[2 * i + 1] = (float) s.getVariance(); + } + + for (int i = 0; i < alphas.size(); ++i) { + weights[3 * i] = (float) alphas.get(i).getMean(); + weights[3 * i + 1] = weights[3 * i]; + weights[3 * i + 2] = weights[3 * i]; + } + + return new Pair(new FloatVectorImpl(vec), weights); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorLayoutDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorLayoutDescriptor.java index ee22a5041..04388085b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorLayoutDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorLayoutDescriptor.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.util; +import java.util.ArrayList; +import java.util.LinkedList; import org.vitrivr.cineast.core.color.ColorConverter; import org.vitrivr.cineast.core.color.RGBContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; @@ -8,86 +10,84 @@ import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.raw.images.MultiImage; -import java.util.ArrayList; -import java.util.LinkedList; - public class ColorLayoutDescriptor { - private ColorLayoutDescriptor() { - } - - private static final int[] scan = { 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, - 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, - 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, - 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, - 63 }; - - private static final double SQRT1_2 = Math.sqrt(0.5); - - - public static FloatVector calculateCLD(MultiImage img) { - - ArrayList tmpList = new ArrayList(img.getWidth() * img.getHeight()); - int[] colors = img.getColors(); - for (int c : colors) { - //set all sufficiently transparent values to white - if(ReadableRGBContainer.getAlpha(c) < 127){ - c = ReadableRGBContainer.WHITE_INT; - } - tmpList.add(c); - } - ArrayList> partitions = GridPartitioner.partition( - tmpList, img.getWidth(), img.getHeight(), 8, 8); - - int[] rgbs = new int[64]; - - for (int i = 0; i < partitions.size(); ++i) { - rgbs[i] = ColorUtils.getAvg(partitions.get(i)); - } - - int[][] ycbcrs = new int[3][64]; - for (int i = 0; i < 64; ++i) { - YCbCrContainer c = ColorConverter.RGBtoYCbCr(new RGBContainer( - rgbs[i])); - ycbcrs[0][i] = c.getY(); - ycbcrs[1][i] = c.getCb(); - ycbcrs[2][i] = c.getCr(); - } - - ycbcrs[0] = dct(ycbcrs[0]); - ycbcrs[1] = dct(ycbcrs[1]); - ycbcrs[2] = dct(ycbcrs[2]); - - float[] cld = new float[]{ - ycbcrs[0][0], ycbcrs[0][1], ycbcrs[0][2], ycbcrs[0][3], ycbcrs[0][4], ycbcrs[0][5], - ycbcrs[1][0], ycbcrs[1][1], ycbcrs[1][2], - ycbcrs[2][0], ycbcrs[2][1], ycbcrs[2][2] - }; - - return new FloatVectorImpl(cld); - - } - /* based on c implementation by Berk ATABEK (http://www.batabek.com/) */ - private static int[] dct(int[] block) { - double sum, cu, cv; - int[] temp = new int[64]; - - for (int u = 0; u < 8; ++u) { - for (int v = 0; v < 8; ++v) { - sum = 0.0; - cu = (u == 0) ? SQRT1_2 : 1.0; - cv = (v == 0) ? SQRT1_2 : 1.0; - for (int x = 0; x < 8; ++x) { - for (int y = 0; y < 8; ++y) { - sum += block[x * 8 + y] - * Math.cos((2 * x + 1) * u * Math.PI / 16.0) - * Math.cos((2 * y + 1) * v * Math.PI / 16.0); - } - } - temp[scan[8 * u + v]] = (int) Math.floor((0.25 * cu * cv * sum) + 0.5); - } - } - return temp; - } - + private ColorLayoutDescriptor() { + } + + private static final int[] scan = {0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, + 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, + 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, + 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, + 63}; + + private static final double SQRT1_2 = Math.sqrt(0.5); + + + public static FloatVector calculateCLD(MultiImage img) { + + ArrayList tmpList = new ArrayList(img.getWidth() * img.getHeight()); + int[] colors = img.getColors(); + for (int c : colors) { + //set all sufficiently transparent values to white + if (ReadableRGBContainer.getAlpha(c) < 127) { + c = ReadableRGBContainer.WHITE_INT; + } + tmpList.add(c); + } + ArrayList> partitions = GridPartitioner.partition( + tmpList, img.getWidth(), img.getHeight(), 8, 8); + + int[] rgbs = new int[64]; + + for (int i = 0; i < partitions.size(); ++i) { + rgbs[i] = ColorUtils.getAvg(partitions.get(i)); + } + + int[][] ycbcrs = new int[3][64]; + for (int i = 0; i < 64; ++i) { + YCbCrContainer c = ColorConverter.RGBtoYCbCr(new RGBContainer( + rgbs[i])); + ycbcrs[0][i] = c.getY(); + ycbcrs[1][i] = c.getCb(); + ycbcrs[2][i] = c.getCr(); + } + + ycbcrs[0] = dct(ycbcrs[0]); + ycbcrs[1] = dct(ycbcrs[1]); + ycbcrs[2] = dct(ycbcrs[2]); + + float[] cld = new float[]{ + ycbcrs[0][0], ycbcrs[0][1], ycbcrs[0][2], ycbcrs[0][3], ycbcrs[0][4], ycbcrs[0][5], + ycbcrs[1][0], ycbcrs[1][1], ycbcrs[1][2], + ycbcrs[2][0], ycbcrs[2][1], ycbcrs[2][2] + }; + + return new FloatVectorImpl(cld); + + } + + /* based on c implementation by Berk ATABEK (http://www.batabek.com/) */ + private static int[] dct(int[] block) { + double sum, cu, cv; + int[] temp = new int[64]; + + for (int u = 0; u < 8; ++u) { + for (int v = 0; v < 8; ++v) { + sum = 0.0; + cu = (u == 0) ? SQRT1_2 : 1.0; + cv = (v == 0) ? SQRT1_2 : 1.0; + for (int x = 0; x < 8; ++x) { + for (int y = 0; y < 8; ++y) { + sum += block[x * 8 + y] + * Math.cos((2 * x + 1) * u * Math.PI / 16.0) + * Math.cos((2 * y + 1) * v * Math.PI / 16.0); + } + } + temp[scan[8 * u + v]] = (int) Math.floor((0.25 * cu * cv * sum) + 0.5); + } + } + return temp; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorReductionUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorReductionUtil.java index 21c9ed406..9b67f8414 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorReductionUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorReductionUtil.java @@ -7,8 +7,9 @@ public class ColorReductionUtil { - private ColorReductionUtil(){} - + private ColorReductionUtil() { + } + private enum Color11 { RED(0xFF0000), ORANGE(0xFFAA00), YELLOW(0xFFFF00), GREEN(0x00FF00), CYAN(0x00FFFF), BLUE( 0x0000FF), VIOLET(0xAA00FF), PURPLE(0xFF00AA), WHITE(0xFFFFFF), GREY(0x808080), BLACK(0); @@ -20,37 +21,37 @@ private enum Color11 { } } - - public static int quantize15(int rgb){ + + public static int quantize15(int rgb) { return FuzzyColorHistogramQuantizer.quantize(ColorConverter.cachedRGBtoLab(rgb)).getRGB().toIntColor(); } - - public static MultiImage quantize15(MultiImage img){ + + public static MultiImage quantize15(MultiImage img) { int[] inColors = img.getColors(); int[] outColors = new int[inColors.length]; - - for(int i = 0; i< inColors.length; ++i){ + + for (int i = 0; i < inColors.length; ++i) { outColors[i] = quantize15(inColors[i]); } - + return img.factory().newMultiImage(img.getWidth(), img.getHeight(), outColors); } - - public static int quantize11(int rgb){ + + public static int quantize11(int rgb) { return quantize11(ColorConverter.cachedRGBtoHSV(rgb)).color; } - - public static MultiImage quantize11(MultiImage img){ + + public static MultiImage quantize11(MultiImage img) { int[] inColors = img.getColors(); int[] outColors = new int[inColors.length]; - - for(int i = 0; i< inColors.length; ++i){ + + for (int i = 0; i < inColors.length; ++i) { outColors[i] = quantize11(inColors[i]); } - + return img.factory().newMultiImage(img.getWidth(), img.getHeight(), outColors); } - + private static Color11 quantize11(ReadableHSVContainer hsv) { if (hsv.getV() < 0.25f) { return Color11.BLACK; @@ -63,10 +64,10 @@ private static Color11 quantize11(ReadableHSVContainer hsv) { } } - if(hsv.getS() * hsv.getV() < 0.1f){ + if (hsv.getS() * hsv.getV() < 0.1f) { return Color11.GREY; } - + float angle = hsv.getH() * 360f; if (angle > 25f && angle <= 50f) { @@ -86,5 +87,5 @@ private static Color11 quantize11(ReadableHSVContainer hsv) { } return Color11.RED; } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java index 366cdb4be..70d095a85 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ColorUtils.java @@ -1,133 +1,134 @@ package org.vitrivr.cineast.core.util; +import java.util.List; import org.vitrivr.cineast.core.color.LabContainer; import org.vitrivr.cineast.core.color.ReadableLabContainer; import org.vitrivr.cineast.core.color.ReadableRGBContainer; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.ReadableFloatVector; -import java.util.List; - public class ColorUtils { - private ColorUtils(){} - - public static final ReadableLabContainer getAvg(List containers){ - int size = containers.size(); - if(size == 0){ - return new LabContainer(0, 0, 0); - } - double L = 0, a = 0, b = 0, alpha = 0; - for(ReadableLabContainer container : containers){ - L += container.getL() * container.getAlpha(); - a += container.getA() * container.getAlpha(); - b += container.getB() * container.getAlpha(); - alpha += container.getAlpha(); - } - if(alpha < 1f){ - return new LabContainer(0, 0, 0); - } - return new LabContainer(L / alpha, a / alpha, b / alpha); - } - - public static final int getAvg(int[] colors){ - - if(colors.length == 0){ - return ReadableRGBContainer.WHITE_INT; - } - - float r = 0, g = 0, b = 0, a = 0, len = 0; - for(int color : colors){ - a = ReadableRGBContainer.getAlpha(color) / 255f; - r += ReadableRGBContainer.getRed(color) * a; - g += ReadableRGBContainer.getGreen(color) * a; - b += ReadableRGBContainer.getBlue(color) * a; - len += a; - } - - if(len < 1f){ - return ReadableRGBContainer.WHITE_INT; - } - - return ReadableRGBContainer.toIntColor(Math.round(r / len), Math.round(g / len), Math.round(b / len)); - } - - public static final int median(Iterable colors){ - int[] histR = new int[256], histG = new int[256], histB = new int[256]; - for(int c : colors){ - if(ReadableRGBContainer.getAlpha(c) < 127){ - continue; - } - histR[ReadableRGBContainer.getRed(c)]++; - histG[ReadableRGBContainer.getGreen(c)]++; - histB[ReadableRGBContainer.getBlue(c)]++; - } - return ReadableRGBContainer.toIntColor(medianFromHistogram(histR), medianFromHistogram(histG), medianFromHistogram(histB)); - } - - private static int medianFromHistogram(int[] hist){ - int pos_l = 0, pos_r = hist.length - 1; - int sum_l = hist[pos_l], sum_r = hist[pos_r]; - - while(pos_l < pos_r){ - if(sum_l < sum_r){ - sum_l += hist[++pos_l]; - }else{ - sum_r += hist[--pos_r]; - } - } - return pos_l; - } - - public static final int getAvg(Iterable colors){ - - float r = 0, g = 0, b = 0, a = 0, len = 0; - for(int color : colors){ - a = ReadableRGBContainer.getAlpha(color) / 255f; - r += ReadableRGBContainer.getRed(color) * a; - g += ReadableRGBContainer.getGreen(color) * a; - b += ReadableRGBContainer.getBlue(color) * a; - len += a; - } - - if(len < 1){ - return ReadableRGBContainer.WHITE_INT; - } - - return ReadableRGBContainer.toIntColor(Math.round(r / len), Math.round(g / len), Math.round(b / len)); - } - - /** - * @return value between 0 and 1 - */ - public static final float getAvgAlpha(Iterable colors){ - float a = 0f; - int count = 0; - for(int color : colors){ - a += ReadableRGBContainer.getAlpha(color) / 255f; - ++count; - } - return a / count; - } - - public static final FloatVector getAvg(List vectors, FloatVector result){ - int size = vectors.size(); - if(size == 0){ - for(int i = 0; i < result.getElementCount(); ++i){ - result.setElement(i, 0f); - } - return result; - } - double[] sum = new double[result.getElementCount()]; - for(ReadableFloatVector vector : vectors){ - for(int i = 0; i < sum.length; ++i){ - sum[i] += vector.getElement(i); - } - } - - for(int i = 0; i < sum.length; ++i){ - result.setElement(i, (float) (sum[i] / size)); - } - - return result; - } + + private ColorUtils() { + } + + public static final ReadableLabContainer getAvg(List containers) { + int size = containers.size(); + if (size == 0) { + return new LabContainer(0, 0, 0); + } + double L = 0, a = 0, b = 0, alpha = 0; + for (ReadableLabContainer container : containers) { + L += container.getL() * container.getAlpha(); + a += container.getA() * container.getAlpha(); + b += container.getB() * container.getAlpha(); + alpha += container.getAlpha(); + } + if (alpha < 1f) { + return new LabContainer(0, 0, 0); + } + return new LabContainer(L / alpha, a / alpha, b / alpha); + } + + public static final int getAvg(int[] colors) { + + if (colors.length == 0) { + return ReadableRGBContainer.WHITE_INT; + } + + float r = 0, g = 0, b = 0, a = 0, len = 0; + for (int color : colors) { + a = ReadableRGBContainer.getAlpha(color) / 255f; + r += ReadableRGBContainer.getRed(color) * a; + g += ReadableRGBContainer.getGreen(color) * a; + b += ReadableRGBContainer.getBlue(color) * a; + len += a; + } + + if (len < 1f) { + return ReadableRGBContainer.WHITE_INT; + } + + return ReadableRGBContainer.toIntColor(Math.round(r / len), Math.round(g / len), Math.round(b / len)); + } + + public static final int median(Iterable colors) { + int[] histR = new int[256], histG = new int[256], histB = new int[256]; + for (int c : colors) { + if (ReadableRGBContainer.getAlpha(c) < 127) { + continue; + } + histR[ReadableRGBContainer.getRed(c)]++; + histG[ReadableRGBContainer.getGreen(c)]++; + histB[ReadableRGBContainer.getBlue(c)]++; + } + return ReadableRGBContainer.toIntColor(medianFromHistogram(histR), medianFromHistogram(histG), medianFromHistogram(histB)); + } + + private static int medianFromHistogram(int[] hist) { + int pos_l = 0, pos_r = hist.length - 1; + int sum_l = hist[pos_l], sum_r = hist[pos_r]; + + while (pos_l < pos_r) { + if (sum_l < sum_r) { + sum_l += hist[++pos_l]; + } else { + sum_r += hist[--pos_r]; + } + } + return pos_l; + } + + public static final int getAvg(Iterable colors) { + + float r = 0, g = 0, b = 0, a = 0, len = 0; + for (int color : colors) { + a = ReadableRGBContainer.getAlpha(color) / 255f; + r += ReadableRGBContainer.getRed(color) * a; + g += ReadableRGBContainer.getGreen(color) * a; + b += ReadableRGBContainer.getBlue(color) * a; + len += a; + } + + if (len < 1) { + return ReadableRGBContainer.WHITE_INT; + } + + return ReadableRGBContainer.toIntColor(Math.round(r / len), Math.round(g / len), Math.round(b / len)); + } + + /** + * @return value between 0 and 1 + */ + public static final float getAvgAlpha(Iterable colors) { + float a = 0f; + int count = 0; + for (int color : colors) { + a += ReadableRGBContainer.getAlpha(color) / 255f; + ++count; + } + return a / count; + } + + public static final FloatVector getAvg(List vectors, FloatVector result) { + int size = vectors.size(); + if (size == 0) { + for (int i = 0; i < result.getElementCount(); ++i) { + result.setElement(i, 0f); + } + return result; + } + double[] sum = new double[result.getElementCount()]; + for (ReadableFloatVector vector : vectors) { + for (int i = 0; i < sum.length; ++i) { + sum[i] += vector.getElement(i); + } + } + + for (int i = 0; i < sum.length; ++i) { + result.setElement(i, (float) (sum[i] / size)); + } + + return result; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/DecodingError.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/DecodingError.java index ec59b759b..43cabe014 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/DecodingError.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/DecodingError.java @@ -1,8 +1,8 @@ package org.vitrivr.cineast.core.util; -public class DecodingError extends RuntimeException{ +public class DecodingError extends RuntimeException { - private static final long serialVersionUID = -3064957319544361747L; + private static final long serialVersionUID = -3064957319544361747L; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GridPartitioner.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GridPartitioner.java index 2cd2a25d1..4d7bf8877 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GridPartitioner.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GridPartitioner.java @@ -6,22 +6,23 @@ public class GridPartitioner { - private GridPartitioner(){} - - public static ArrayList> partition(List input, int width, int height, int xpartitions, int ypartitions){ - ArrayList> _return = new ArrayList>(xpartitions * ypartitions); - for(int i = 0; i < xpartitions * ypartitions; ++i){ - _return.add(new LinkedList()); - } - - int i = 0; - for(T t : input){ - int index = (((i % width) * xpartitions) / width) + xpartitions * (i * ypartitions / width / height); - _return.get(index).add(t); - ++i; - } - - return _return; - } - + private GridPartitioner() { + } + + public static ArrayList> partition(List input, int width, int height, int xpartitions, int ypartitions) { + ArrayList> _return = new ArrayList>(xpartitions * ypartitions); + for (int i = 0; i < xpartitions * ypartitions; ++i) { + _return.add(new LinkedList()); + } + + int i = 0; + for (T t : input) { + int index = (((i % width) * xpartitions) / width) + xpartitions * (i * ypartitions / width / height); + _return.get(index).add(t); + ++i; + } + + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GroupingUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GroupingUtil.java index c5f8052d9..b654232ef 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GroupingUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/GroupingUtil.java @@ -1,22 +1,27 @@ package org.vitrivr.cineast.core.util; import com.google.common.collect.ImmutableList; - -import java.util.*; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Stream; public class GroupingUtil { - private GroupingUtil() {} + + private GroupingUtil() { + } /** * Filters the given {@code elements} by keeping only the greatest element of each group. * - * @param elements stream of elements to filter + * @param elements stream of elements to filter * @param groupFunction function that returns the group attribute given an element - * @param comparator comparator to determine the greatest element - * @param type of elements - * @param type of the group attribute + * @param comparator comparator to determine the greatest element + * @param type of elements + * @param type of the group attribute * @return list of elements containing the maximum element of each group */ public static List filterMaxByGroup(Stream elements, Function groupFunction, Comparator comparator) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/HungarianAlgorithm.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/HungarianAlgorithm.java index eb9101f6e..5817d038b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/HungarianAlgorithm.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/HungarianAlgorithm.java @@ -36,8 +36,7 @@ */ /** - * The Hungarian Algorithm takes a cost matrix, and finds the optimal minimum one-to-one matches between row and column elements. - * The algorithm is particularly useful for multi-object tracking in which detections from one frame have to be matched with detections in an adjacent frame + * The Hungarian Algorithm takes a cost matrix, and finds the optimal minimum one-to-one matches between row and column elements. The algorithm is particularly useful for multi-object tracking in which detections from one frame have to be matched with detections in an adjacent frame */ public class HungarianAlgorithm { @@ -53,11 +52,7 @@ public class HungarianAlgorithm { /** * Construct an instance of the algorithm. * - * @param costMatrix - * the cost matrix, where matrix[i][j] holds the cost of assigning - * worker i to job j, for all i, j. The cost matrix must not be - * irregular in the sense that all rows must be the same length; in - * addition, all entries must be non-infinite numbers. + * @param costMatrix the cost matrix, where matrix[i][j] holds the cost of assigning worker i to job j, for all i, j. The cost matrix must not be irregular in the sense that all rows must be the same length; in addition, all entries must be non-infinite numbers. */ public HungarianAlgorithm(double[][] costMatrix) { this.dim = Math.max(costMatrix.length, costMatrix[0].length); @@ -95,9 +90,7 @@ public HungarianAlgorithm(double[][] costMatrix) { } /** - * Compute an initial feasible solution by assigning zero labels to the - * workers and by assigning to each job a label equal to the minimum cost - * among its incident edges. + * Compute an initial feasible solution by assigning zero labels to the workers and by assigning to each job a label equal to the minimum cost among its incident edges. */ protected void computeInitialFeasibleSolution() { for (int j = 0; j < dim; j++) { @@ -115,9 +108,7 @@ protected void computeInitialFeasibleSolution() { /** * Execute the algorithm. * - * @return the minimum cost matching of workers to jobs based upon the - * provided cost matrix. A matching value of -1 indicates that the - * corresponding worker is unassigned. + * @return the minimum cost matching of workers to jobs based upon the provided cost matrix. A matching value of -1 indicates that the corresponding worker is unassigned. */ public int[] execute() { /* @@ -145,22 +136,10 @@ public int[] execute() { } /** - * Execute a single phase of the algorithm. A phase of the Hungarian algorithm - * consists of building a set of committed workers and a set of committed jobs - * from a root unmatched worker by following alternating unmatched/matched - * zero-slack edges. If an unmatched job is encountered, then an augmenting - * path has been found and the matching is grown. If the connected zero-slack - * edges have been exhausted, the labels of committed workers are increased by - * the minimum slack among committed workers and non-committed jobs to create - * more zero-slack edges (the labels of committed jobs are simultaneously - * decreased by the same amount in order to maintain a feasible labeling). + * Execute a single phase of the algorithm. A phase of the Hungarian algorithm consists of building a set of committed workers and a set of committed jobs from a root unmatched worker by following alternating unmatched/matched zero-slack edges. If an unmatched job is encountered, then an augmenting path has been found and the matching is grown. If the connected zero-slack edges have been exhausted, the labels of committed workers are increased by the minimum slack among committed workers and non-committed jobs to create more zero-slack edges (the labels of committed jobs are simultaneously decreased by the same amount in order to maintain a feasible labeling). *

    - * - * The runtime of a single phase of the algorithm is O(n^2), where n is the - * dimension of the internal square cost matrix, since each edge is visited at - * most once and since increasing the labeling is accomplished in time O(n) by - * maintaining the minimum slack values among non-committed jobs. When a phase - * completes, the matching will have increased in size. + *

    + * The runtime of a single phase of the algorithm is O(n^2), where n is the dimension of the internal square cost matrix, since each edge is visited at most once and since increasing the labeling is accomplished in time O(n) by maintaining the minimum slack values among non-committed jobs. When a phase completes, the matching will have increased in size. */ protected void executePhase() { while (true) { @@ -217,7 +196,6 @@ protected void executePhase() { } /** - * * @return the first unmatched worker or {@link #dim} if none. */ protected int fetchUnmatchedWorker() { @@ -231,8 +209,7 @@ protected int fetchUnmatchedWorker() { } /** - * Find a valid matching by greedily selecting among zero-cost matchings. This - * is a heuristic to jump-start the augmentation algorithm. + * Find a valid matching by greedily selecting among zero-cost matchings. This is a heuristic to jump-start the augmentation algorithm. */ protected void greedyMatch() { for (int w = 0; w < dim; w++) { @@ -246,12 +223,9 @@ protected void greedyMatch() { } /** - * Initialize the next phase of the algorithm by clearing the committed - * workers and jobs sets and by initializing the slack arrays to the values - * corresponding to the specified root worker. + * Initialize the next phase of the algorithm by clearing the committed workers and jobs sets and by initializing the slack arrays to the values corresponding to the specified root worker. * - * @param w - * the worker at which to root the next phase. + * @param w the worker at which to root the next phase. */ protected void initializePhase(int w) { Arrays.fill(committedWorkers, false); @@ -273,10 +247,7 @@ protected void match(int w, int j) { } /** - * Reduce the cost matrix by subtracting the smallest element of each row from - * all elements of the row as well as the smallest element of each column from - * all elements of the column. Note that an optimal assignment for a reduced - * cost matrix is optimal for the original cost matrix. + * Reduce the cost matrix by subtracting the smallest element of each row from all elements of the row as well as the smallest element of each column from all elements of the column. Note that an optimal assignment for a reduced cost matrix is optimal for the original cost matrix. */ protected void reduce() { for (int w = 0; w < dim; w++) { @@ -309,9 +280,7 @@ protected void reduce() { } /** - * Update labels with the specified slack by adding the slack value for - * committed workers and by subtracting the slack value for committed jobs. In - * addition, update the minimum slack values appropriately. + * Update labels with the specified slack by adding the slack value for committed workers and by subtracting the slack value for committed jobs. In addition, update the minimum slack values appropriately. */ protected void updateLabeling(double slack) { for (int w = 0; w < dim; w++) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java index 2f0c76168..b23cdd515 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ImageHistogramEqualizer.java @@ -3,90 +3,89 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.vitrivr.cineast.core.color.ReadableRGBContainer; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; - import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.vitrivr.cineast.core.color.ReadableRGBContainer; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; public class ImageHistogramEqualizer { - private ImageHistogramEqualizer(){} - - private static LoadingCache cache = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES).build(new CacheLoader(){ - - @Override - public MultiImage load(MultiImage in) throws Exception { - return equalizeHistogram(in); - } - }); - - public static MultiImage getEqualized(MultiImage in){ - try { - synchronized(cache){ - return cache.get(in); - } - } catch (ExecutionException e) { - return equalizeHistogram(in); //should never happen - } - } - - /** - * - * Equalizes the color histogram of the input image. - */ - private static MultiImage equalizeHistogram(MultiImage in){ - int[] inColors = in.getColors(); - int[] returnColors = new int[inColors.length]; - - float[] red = new float[256], green = new float[256], blue = new float[256]; - - // build histogram - for(int color : inColors){ - float alpha = ReadableRGBContainer.getAlpha(color) / 255f; - red[ReadableRGBContainer.getRed(color)] += alpha; - green[ReadableRGBContainer.getGreen(color)] += alpha; - blue[ReadableRGBContainer.getBlue(color)] += alpha; - } - - int[] redMap = buildMapFromHist(red); - int[] greenMap = buildMapFromHist(green); - int[] blueMap = buildMapFromHist(blue); - - //apply mapping - for(int i = 0; i < inColors.length; ++i){ - int color = inColors[i]; - returnColors[i] = ReadableRGBContainer.toIntColor( - redMap[ReadableRGBContainer.getRed(color)], - greenMap[ReadableRGBContainer.getGreen(color)], - blueMap[ReadableRGBContainer.getBlue(color)], - ReadableRGBContainer.getAlpha(color)); - } - - return in.factory().newMultiImage(in.getWidth(), in.getHeight(), returnColors); - - } - - private static int[] buildMapFromHist(float[] hist){ - int[] _return = new int[hist.length]; - _return[0] = Math.round(hist[0]); - - for(int i = 1; i < hist.length; ++i){ - _return[i] = Math.round(_return[i-1] + hist[i]); - } - - if(_return[0] == _return[_return.length - 1]){//all zeros - for(int i = 0; i < _return.length; ++i){ - _return[i] = 0; - } - return _return; - } - - for(int i = 0; i < hist.length; ++i){ - _return[i] = Math.round((_return[i] * 255.0f) / _return[_return.length - 1]); - } - - return _return; - } - + private ImageHistogramEqualizer() { + } + + private static LoadingCache cache = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES).build(new CacheLoader() { + + @Override + public MultiImage load(MultiImage in) throws Exception { + return equalizeHistogram(in); + } + }); + + public static MultiImage getEqualized(MultiImage in) { + try { + synchronized (cache) { + return cache.get(in); + } + } catch (ExecutionException e) { + return equalizeHistogram(in); //should never happen + } + } + + /** + * Equalizes the color histogram of the input image. + */ + private static MultiImage equalizeHistogram(MultiImage in) { + int[] inColors = in.getColors(); + int[] returnColors = new int[inColors.length]; + + float[] red = new float[256], green = new float[256], blue = new float[256]; + + // build histogram + for (int color : inColors) { + float alpha = ReadableRGBContainer.getAlpha(color) / 255f; + red[ReadableRGBContainer.getRed(color)] += alpha; + green[ReadableRGBContainer.getGreen(color)] += alpha; + blue[ReadableRGBContainer.getBlue(color)] += alpha; + } + + int[] redMap = buildMapFromHist(red); + int[] greenMap = buildMapFromHist(green); + int[] blueMap = buildMapFromHist(blue); + + //apply mapping + for (int i = 0; i < inColors.length; ++i) { + int color = inColors[i]; + returnColors[i] = ReadableRGBContainer.toIntColor( + redMap[ReadableRGBContainer.getRed(color)], + greenMap[ReadableRGBContainer.getGreen(color)], + blueMap[ReadableRGBContainer.getBlue(color)], + ReadableRGBContainer.getAlpha(color)); + } + + return in.factory().newMultiImage(in.getWidth(), in.getHeight(), returnColors); + + } + + private static int[] buildMapFromHist(float[] hist) { + int[] _return = new int[hist.length]; + _return[0] = Math.round(hist[0]); + + for (int i = 1; i < hist.length; ++i) { + _return[i] = Math.round(_return[i - 1] + hist[i]); + } + + if (_return[0] == _return[_return.length - 1]) {//all zeros + for (int i = 0; i < _return.length; ++i) { + _return[i] = 0; + } + return _return; + } + + for (int i = 0; i < hist.length; ++i) { + _return[i] = Math.round((_return[i] * 255.0f) / _return[_return.length - 1]); + } + + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/KMeansPP.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/KMeansPP.java index 2d4715d56..6d7fef10d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/KMeansPP.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/KMeansPP.java @@ -1,17 +1,21 @@ package org.vitrivr.cineast.core.util; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Random; import org.vitrivr.cineast.core.data.FloatVector; import org.vitrivr.cineast.core.data.FloatVectorImpl; import org.vitrivr.cineast.core.data.ReadableFloatVector; -import java.util.*; - public strictfp class KMeansPP { private KMeansPP() { } public static class KMenasResult { + private final int k; private final double[] distances; private double distance; @@ -77,8 +81,7 @@ public double getDistance(int i) { private static final Random random = new Random(1); /** - * performs {@link KMeansPP} runs times and returns the result with the - * minimal overall distance + * performs {@link KMeansPP} runs times and returns the result with the minimal overall distance */ public static KMenasResult bestOfkMeansPP( List elements, FloatVector helper, int k, double minDist, int runs) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/LogHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/LogHelper.java index 3841889a5..a7ee63f52 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/LogHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/LogHelper.java @@ -1,22 +1,21 @@ package org.vitrivr.cineast.core.util; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; - import java.io.PrintWriter; import java.io.StringWriter; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; public class LogHelper { - private LogHelper() { - } + private LogHelper() { + } - public static final Marker SQL_MARKER = MarkerManager.getMarker("SQL"); + public static final Marker SQL_MARKER = MarkerManager.getMarker("SQL"); - public static String getStackTrace(Throwable e){ - StringWriter sWriter = new StringWriter(); - PrintWriter pWriter = new PrintWriter(sWriter); - e.printStackTrace(pWriter); - return sWriter.toString(); - } + public static String getStackTrace(Throwable e) { + StringWriter sWriter = new StringWriter(); + PrintWriter pWriter = new PrintWriter(sWriter); + e.printStackTrace(pWriter); + return sWriter.toString(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MaskGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MaskGenerator.java index ffc9db0f5..e4597adaa 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MaskGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MaskGenerator.java @@ -15,6 +15,11 @@ import boofcv.struct.image.GrayU8; import georegression.struct.point.Point2D_F32; import georegression.struct.point.Point2D_I32; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; import org.ddogleg.nn.FactoryNearestNeighbor; import org.ddogleg.nn.NearestNeighbor; import org.ddogleg.nn.NnData; @@ -23,381 +28,376 @@ import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.descriptor.PathList; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; - public class MaskGenerator { - private MaskGenerator(){} - - public static ArrayList>> getNormalizedBbox(List videoFrames, - List>> foregroundPaths, - List>> backgroundPaths){ - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList>> bboxWithIdx = new ArrayList>>(); - - ArrayList rects = MaskGenerator.getFgBoundingBox(videoFrames, foregroundPaths, backgroundPaths); - - int width = videoFrames.get(0).getImage().getWidth(); - int height = videoFrames.get(0).getImage().getHeight(); - - long frameIdx = 0; - - for(ImageRectangle rect : rects){ - ArrayList bbox = normalize(rect, width, height); - bboxWithIdx.add(new Pair>(frameIdx, bbox)); - frameIdx += PathList.frameInterval; - } - - return bboxWithIdx; - } - - public static ArrayList normalize(ImageRectangle rect, int width, int height){ - ArrayList norm = new ArrayList(); - norm.add((float)rect.getX0() / (float)width); - norm.add((float)rect.getY0() / (float)height); - norm.add((float)rect.getWidth() / (float)width); - norm.add((float)rect.getHeight() / (float)height); - return norm; - } - - public static ArrayList getFgBoundingBox(List videoFrames, - List>> foregroundPaths, - List>> backgroundPaths){ - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList rects = new ArrayList(); - ArrayList masks = getFgMasksByNN(videoFrames, foregroundPaths, backgroundPaths); - - for(GrayU8 mask : masks){ - ImageRectangle rect = getLargestBoundingBox(mask); - rects.add(rect); - } - - return rects; - } - - public static ImageRectangle getLargestBoundingBox(GrayU8 mask){ - - List rects = getBoundingBox(mask); - ImageRectangle largest = new ImageRectangle(0,0,0,0); - for(ImageRectangle rect : rects){ - if(rect.getWidth() * rect.getHeight() > largest.getWidth() * largest.getHeight()){ - largest = rect; - } - } - return largest; - } - - public static List getBoundingBox(GrayU8 mask){ - - List contours = BinaryImageOps.contour(mask,ConnectRule.FOUR,null); - List rects = new ArrayList(); - - for(Contour contour : contours){ - ImageRectangle rect = new ImageRectangle(mask.width,mask.height,0,0); - for (Point2D_I32 p : contour.external){ - if (p.x < rect.x0) { + private MaskGenerator() { + } + + public static ArrayList>> getNormalizedBbox(List videoFrames, + List>> foregroundPaths, + List>> backgroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList>> bboxWithIdx = new ArrayList>>(); + + ArrayList rects = MaskGenerator.getFgBoundingBox(videoFrames, foregroundPaths, backgroundPaths); + + int width = videoFrames.get(0).getImage().getWidth(); + int height = videoFrames.get(0).getImage().getHeight(); + + long frameIdx = 0; + + for (ImageRectangle rect : rects) { + ArrayList bbox = normalize(rect, width, height); + bboxWithIdx.add(new Pair>(frameIdx, bbox)); + frameIdx += PathList.frameInterval; + } + + return bboxWithIdx; + } + + public static ArrayList normalize(ImageRectangle rect, int width, int height) { + ArrayList norm = new ArrayList(); + norm.add((float) rect.getX0() / (float) width); + norm.add((float) rect.getY0() / (float) height); + norm.add((float) rect.getWidth() / (float) width); + norm.add((float) rect.getHeight() / (float) height); + return norm; + } + + public static ArrayList getFgBoundingBox(List videoFrames, + List>> foregroundPaths, + List>> backgroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList rects = new ArrayList(); + ArrayList masks = getFgMasksByNN(videoFrames, foregroundPaths, backgroundPaths); + + for (GrayU8 mask : masks) { + ImageRectangle rect = getLargestBoundingBox(mask); + rects.add(rect); + } + + return rects; + } + + public static ImageRectangle getLargestBoundingBox(GrayU8 mask) { + + List rects = getBoundingBox(mask); + ImageRectangle largest = new ImageRectangle(0, 0, 0, 0); + for (ImageRectangle rect : rects) { + if (rect.getWidth() * rect.getHeight() > largest.getWidth() * largest.getHeight()) { + largest = rect; + } + } + return largest; + } + + public static List getBoundingBox(GrayU8 mask) { + + List contours = BinaryImageOps.contour(mask, ConnectRule.FOUR, null); + List rects = new ArrayList(); + + for (Contour contour : contours) { + ImageRectangle rect = new ImageRectangle(mask.width, mask.height, 0, 0); + for (Point2D_I32 p : contour.external) { + if (p.x < rect.x0) { rect.x0 = p.x; } - if (p.y < rect.y0) { + if (p.y < rect.y0) { rect.y0 = p.y; } - if (p.x > rect.x1) { + if (p.x > rect.x1) { rect.x1 = p.x; } - if (p.y > rect.y1) { + if (p.y > rect.y1) { rect.y1 = p.y; } - } - rects.add(rect); - } - - return rects; - } - - public static ArrayList getFgMasksByFilter(List videoFrames, - List>> foregroundPaths, - List>> backgroundPaths) { - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList masksScaled = generateScaledMasksFromPath(videoFrames, foregroundPaths); - - ArrayList masksScaledSmoothed1 = createNewMasks(masksScaled); - smoothMasks(masksScaled, masksScaledSmoothed1, 4, 2, 64, 26); - - ArrayList masks = scaleUpMasks(masksScaledSmoothed1, videoFrames.get(0).getImage().getBufferedImage().getWidth(), videoFrames.get(0).getImage().getBufferedImage().getHeight()); - - ArrayList masksSmoothed1 = createNewMasks(masks); - smoothMasks(masks, masksSmoothed1, 5, 2, 64, 10); - ArrayList masksSmoothed2 = createNewMasks(masks); - smoothMasks(masksSmoothed1, masksSmoothed2, 5, 2, 64, 10); - - //multiply3D(masksSmoothed2,masksSmoothed2,255); - - return masksSmoothed2; - } - - public static ArrayList getFgMasksByNN(List videoFrames, - List>> foregroundPaths, - List>> backgroundPaths) { - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList masks = generateMasksFromPath(videoFrames, foregroundPaths, backgroundPaths); - - ArrayList masksSmoothed1 = createNewMasks(masks); - smoothMasks(masks, masksSmoothed1, 21, 2, 64, 26); - ArrayList masksSmoothed2 = createNewMasks(masks); - smoothMasks(masksSmoothed1, masksSmoothed2, 11, 2, 64, 26); - - //multiply3D(masksSmoothed2,masksSmoothed2,255); - - return masksSmoothed2; - } - - public static ArrayList generateMasksFromPath(List videoFrames, - List>> foregroundPaths, - List>> backgroundPaths){ - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList masks = new ArrayList(); - - int width = videoFrames.get(0).getImage().getBufferedImage().getWidth(); - int height = videoFrames.get(0).getImage().getBufferedImage().getHeight(); - - ListIterator>> fgPathItor = foregroundPaths.listIterator(); - ListIterator>> bgPathItor = backgroundPaths.listIterator(); - - int cnt = 0; - for (int frameIdx = 0; frameIdx < videoFrames.size(); ++frameIdx) { - if (cnt >= PathList.frameInterval) { - cnt = 0; - continue; - } - cnt += 1; - - GrayU8 mask = new GrayU8(width, height); - - NearestNeighbor nn = FactoryNearestNeighbor.kdtree(); - LinkedList nnPoints = new LinkedList(); - LinkedList nnData = new LinkedList(); - - while (fgPathItor.hasNext()) { - Pair> pair = fgPathItor.next(); - if (pair.first > frameIdx) { + } + rects.add(rect); + } + + return rects; + } + + public static ArrayList getFgMasksByFilter(List videoFrames, + List>> foregroundPaths, + List>> backgroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList masksScaled = generateScaledMasksFromPath(videoFrames, foregroundPaths); + + ArrayList masksScaledSmoothed1 = createNewMasks(masksScaled); + smoothMasks(masksScaled, masksScaledSmoothed1, 4, 2, 64, 26); + + ArrayList masks = scaleUpMasks(masksScaledSmoothed1, videoFrames.get(0).getImage().getBufferedImage().getWidth(), videoFrames.get(0).getImage().getBufferedImage().getHeight()); + + ArrayList masksSmoothed1 = createNewMasks(masks); + smoothMasks(masks, masksSmoothed1, 5, 2, 64, 10); + ArrayList masksSmoothed2 = createNewMasks(masks); + smoothMasks(masksSmoothed1, masksSmoothed2, 5, 2, 64, 10); + + //multiply3D(masksSmoothed2,masksSmoothed2,255); + + return masksSmoothed2; + } + + public static ArrayList getFgMasksByNN(List videoFrames, + List>> foregroundPaths, + List>> backgroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList masks = generateMasksFromPath(videoFrames, foregroundPaths, backgroundPaths); + + ArrayList masksSmoothed1 = createNewMasks(masks); + smoothMasks(masks, masksSmoothed1, 21, 2, 64, 26); + ArrayList masksSmoothed2 = createNewMasks(masks); + smoothMasks(masksSmoothed1, masksSmoothed2, 11, 2, 64, 26); + + //multiply3D(masksSmoothed2,masksSmoothed2,255); + + return masksSmoothed2; + } + + public static ArrayList generateMasksFromPath(List videoFrames, + List>> foregroundPaths, + List>> backgroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList masks = new ArrayList(); + + int width = videoFrames.get(0).getImage().getBufferedImage().getWidth(); + int height = videoFrames.get(0).getImage().getBufferedImage().getHeight(); + + ListIterator>> fgPathItor = foregroundPaths.listIterator(); + ListIterator>> bgPathItor = backgroundPaths.listIterator(); + + int cnt = 0; + for (int frameIdx = 0; frameIdx < videoFrames.size(); ++frameIdx) { + if (cnt >= PathList.frameInterval) { + cnt = 0; + continue; + } + cnt += 1; + + GrayU8 mask = new GrayU8(width, height); + + NearestNeighbor nn = FactoryNearestNeighbor.kdtree(); + LinkedList nnPoints = new LinkedList(); + LinkedList nnData = new LinkedList(); + + while (fgPathItor.hasNext()) { + Pair> pair = fgPathItor.next(); + if (pair.first > frameIdx) { break; } - Point2D_F32 p = pair.second.getFirst(); - double[] point = {p.x * width, p.y * height}; - nnPoints.add(point); - nnData.add(1); - } - - while (bgPathItor.hasNext()) { - Pair> pair = bgPathItor.next(); - if (pair.first > frameIdx) { + Point2D_F32 p = pair.second.getFirst(); + double[] point = {p.x * width, p.y * height}; + nnPoints.add(point); + nnData.add(1); + } + + while (bgPathItor.hasNext()) { + Pair> pair = bgPathItor.next(); + if (pair.first > frameIdx) { break; } - Point2D_F32 p = pair.second.getFirst(); - double[] point = {p.x * width, p.y * height}; - nnPoints.add(point); - nnData.add(0); - } - - nn.init(2); - nn.setPoints(nnPoints, nnData); - - for(int x = 0; x < width; ++x){ - for(int y = 0; y < height; ++y){ - double[] point = {x, y}; - @SuppressWarnings("unchecked") - FastQueue> results = new FastQueue(5,NnData.class,true); - nn.findNearest(point, -1, 5, results); - int sum = 0; - for(NnData r : results.toList()){ - sum += r.data.intValue(); - } - int value = sum > results.size()/2 ? 1 : 0; - mask.set(x, y, value); - } - } - - //showBineryImage(mask); - masks.add(mask); - } - - return masks; - - } - - public static ArrayList generateScaledMasksFromPath(List videoFrames, - List>> foregroundPaths) { - - if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { - return null; - } - - ArrayList masks = new ArrayList(); - - int width = videoFrames.get(0).getImage().getBufferedImage().getWidth(); - int height = videoFrames.get(0).getImage().getBufferedImage().getHeight(); - - ListIterator>> fgPathItor = foregroundPaths.listIterator(); - - int cnt = 0; - for (int frameIdx = 0; frameIdx < videoFrames.size(); ++frameIdx) { - if (cnt >= PathList.frameInterval) { - cnt = 0; - continue; - } - cnt += 1; - - GrayU8 mask = new GrayU8(width / PathList.samplingInterval, height / PathList.samplingInterval); - - while (fgPathItor.hasNext()) { - Pair> pair = fgPathItor.next(); - if (pair.first > frameIdx) { + Point2D_F32 p = pair.second.getFirst(); + double[] point = {p.x * width, p.y * height}; + nnPoints.add(point); + nnData.add(0); + } + + nn.init(2); + nn.setPoints(nnPoints, nnData); + + for (int x = 0; x < width; ++x) { + for (int y = 0; y < height; ++y) { + double[] point = {x, y}; + @SuppressWarnings("unchecked") + FastQueue> results = new FastQueue(5, NnData.class, true); + nn.findNearest(point, -1, 5, results); + int sum = 0; + for (NnData r : results.toList()) { + sum += r.data.intValue(); + } + int value = sum > results.size() / 2 ? 1 : 0; + mask.set(x, y, value); + } + } + + //showBineryImage(mask); + masks.add(mask); + } + + return masks; + + } + + public static ArrayList generateScaledMasksFromPath(List videoFrames, + List>> foregroundPaths) { + + if (videoFrames == null || videoFrames.isEmpty() || foregroundPaths == null) { + return null; + } + + ArrayList masks = new ArrayList(); + + int width = videoFrames.get(0).getImage().getBufferedImage().getWidth(); + int height = videoFrames.get(0).getImage().getBufferedImage().getHeight(); + + ListIterator>> fgPathItor = foregroundPaths.listIterator(); + + int cnt = 0; + for (int frameIdx = 0; frameIdx < videoFrames.size(); ++frameIdx) { + if (cnt >= PathList.frameInterval) { + cnt = 0; + continue; + } + cnt += 1; + + GrayU8 mask = new GrayU8(width / PathList.samplingInterval, height / PathList.samplingInterval); + + while (fgPathItor.hasNext()) { + Pair> pair = fgPathItor.next(); + if (pair.first > frameIdx) { break; } - Point2D_F32 p1 = pair.second.getFirst(); - int x = (int) (p1.x * width / PathList.samplingInterval); - int y = (int) (p1.y * height / PathList.samplingInterval); - if (mask.isInBounds(x, y)) { - mask.set(x, y, 1); - } - } - //showBineryImage(mask); - masks.add(mask); - } - - return masks; - } - - public static ArrayList smoothMasks(ArrayList input, ArrayList output, - int spatialRadius, int temporalRadius, double multipyFactor, int threshold){ - - if(input == null || input.isEmpty()){ - return input; - } - if(output == null){ - output = createNewMasks(input); - } - if(output.size() != input.size()){ - throw new IllegalArgumentException("size of input and output do not match. input: "+input.size()+" output: "+output.size()); - } - - multiply3D(input, input, multipyFactor); - gaussianFilter3D(input, output, spatialRadius, temporalRadius); - threshold3D(output,output,threshold); - - return output; - } - - public static ArrayList gaussianFilter3D(ArrayList input, ArrayList output, - int spatialRadius, int temporalRadius) { - - ArrayList spatialResult = createNewMasks(input); - gaussianFilterSpatial(input, spatialResult, spatialRadius); - gaussianFilterTemporal(spatialResult, output, temporalRadius); - - return output; - } - - public static ArrayList gaussianFilterSpatial(ArrayList input, ArrayList output, int spatialRadius){ - - for (int i = 0; i < input.size(); ++i){ - GBlurImageOps.gaussian(input.get(i), output.get(i), -1, spatialRadius, null); - } - - return output; - } - - public static ArrayList gaussianFilterTemporal(ArrayList input, ArrayList output, int spatialRadius){ - int width = input.get(0).getWidth(); - int height = input.get(0).getHeight(); - int len = input.size(); - - Kernel1D_F32 kernel = FactoryKernelGaussian.gaussian(Kernel1D_F32.class,-1,spatialRadius); - float divisor = kernel.computeSum(); - int data1D[] = new int[len + 2*kernel.offset]; - for (int x = 0; x < width; ++x){ - for (int y = 0; y < height; ++y){ - for(int i = 0; i < len; ++i){ - data1D[i + kernel.offset] = input.get(i).get(x, y); - } - for(int i = 0; i < len; ++i){ - int total = 0; - for (int k = 0; k < kernel.width; ++k){ - total += (data1D[i+k] & 0xFF) * kernel.data[k]; - } - output.get(i).set(x, y, Math.round(total/divisor)); - } - } - } - - return output; - } - - public static ArrayList multiply3D(ArrayList input, ArrayList output, double value){ - - for (int i = 0; i < input.size(); ++i){ - PixelMath.multiply(input.get(i), value, output.get(i)); - } - - return output; - } - - public static ArrayList threshold3D(ArrayList input, ArrayList output, int threshold){ - - for (int i = 0; i < input.size(); ++i){ - ThresholdImageOps.threshold(input.get(i), output.get(i), threshold, false); - } - - return output; - } - - public static ArrayList createNewMasks(ArrayList input){ - ArrayList output = new ArrayList(); - for (int i = 0; i < input.size(); ++i){ - output.add(input.get(i).createSameShape()); - } - return output; - } - - public static ArrayList scaleUpMasks(ArrayList input, int width, int height){ - ArrayList output = new ArrayList(); - for (int i = 0; i < input.size(); ++i){ - GrayU8 in = input.get(i); - GrayU8 out = new GrayU8(width, height); - new FDistort(in, out).scaleExt().apply(); - output.add(out); - } - return output; - } - - public static void showBineryImage(GrayU8 image){ - PixelMath.multiply(image,255,image); - BufferedImage out = ConvertBufferedImage.convertTo(image,null); - ShowImages.showWindow(out,"Output"); - } - + Point2D_F32 p1 = pair.second.getFirst(); + int x = (int) (p1.x * width / PathList.samplingInterval); + int y = (int) (p1.y * height / PathList.samplingInterval); + if (mask.isInBounds(x, y)) { + mask.set(x, y, 1); + } + } + //showBineryImage(mask); + masks.add(mask); + } + + return masks; + } + + public static ArrayList smoothMasks(ArrayList input, ArrayList output, + int spatialRadius, int temporalRadius, double multipyFactor, int threshold) { + + if (input == null || input.isEmpty()) { + return input; + } + if (output == null) { + output = createNewMasks(input); + } + if (output.size() != input.size()) { + throw new IllegalArgumentException("size of input and output do not match. input: " + input.size() + " output: " + output.size()); + } + + multiply3D(input, input, multipyFactor); + gaussianFilter3D(input, output, spatialRadius, temporalRadius); + threshold3D(output, output, threshold); + + return output; + } + + public static ArrayList gaussianFilter3D(ArrayList input, ArrayList output, + int spatialRadius, int temporalRadius) { + + ArrayList spatialResult = createNewMasks(input); + gaussianFilterSpatial(input, spatialResult, spatialRadius); + gaussianFilterTemporal(spatialResult, output, temporalRadius); + + return output; + } + + public static ArrayList gaussianFilterSpatial(ArrayList input, ArrayList output, int spatialRadius) { + + for (int i = 0; i < input.size(); ++i) { + GBlurImageOps.gaussian(input.get(i), output.get(i), -1, spatialRadius, null); + } + + return output; + } + + public static ArrayList gaussianFilterTemporal(ArrayList input, ArrayList output, int spatialRadius) { + int width = input.get(0).getWidth(); + int height = input.get(0).getHeight(); + int len = input.size(); + + Kernel1D_F32 kernel = FactoryKernelGaussian.gaussian(Kernel1D_F32.class, -1, spatialRadius); + float divisor = kernel.computeSum(); + int data1D[] = new int[len + 2 * kernel.offset]; + for (int x = 0; x < width; ++x) { + for (int y = 0; y < height; ++y) { + for (int i = 0; i < len; ++i) { + data1D[i + kernel.offset] = input.get(i).get(x, y); + } + for (int i = 0; i < len; ++i) { + int total = 0; + for (int k = 0; k < kernel.width; ++k) { + total += (data1D[i + k] & 0xFF) * kernel.data[k]; + } + output.get(i).set(x, y, Math.round(total / divisor)); + } + } + } + + return output; + } + + public static ArrayList multiply3D(ArrayList input, ArrayList output, double value) { + + for (int i = 0; i < input.size(); ++i) { + PixelMath.multiply(input.get(i), value, output.get(i)); + } + + return output; + } + + public static ArrayList threshold3D(ArrayList input, ArrayList output, int threshold) { + + for (int i = 0; i < input.size(); ++i) { + ThresholdImageOps.threshold(input.get(i), output.get(i), threshold, false); + } + + return output; + } + + public static ArrayList createNewMasks(ArrayList input) { + ArrayList output = new ArrayList(); + for (int i = 0; i < input.size(); ++i) { + output.add(input.get(i).createSameShape()); + } + return output; + } + + public static ArrayList scaleUpMasks(ArrayList input, int width, int height) { + ArrayList output = new ArrayList(); + for (int i = 0; i < input.size(); ++i) { + GrayU8 in = input.get(i); + GrayU8 out = new GrayU8(width, height); + new FDistort(in, out).scaleExt().apply(); + output.add(out); + } + return output; + } + + public static void showBineryImage(GrayU8 image) { + PixelMath.multiply(image, 255, image); + BufferedImage out = ConvertBufferedImage.convertTo(image, null); + ShowImages.showWindow(out, "Output"); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MathHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MathHelper.java index 7b1e2b47b..f7a12dbd1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MathHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MathHelper.java @@ -4,280 +4,279 @@ public class MathHelper { - private MathHelper(){} - - public static final double SQRT2 = Math.sqrt(2); - public static final float SQRT2_f = (float)SQRT2; - - public static double getScore(double dist, double maxDist){ - if(Double.isNaN(dist) || Double.isNaN(maxDist)){ - return 0d; - } - double score = 1d - (dist / maxDist); - if(score > 1d){ - return 1d; - } - if(score < 0d){ - return 0d; - } - return score; - } - - /** - * Normalizes a float array with respect to the L2 (euclidian) norm. The - * method will return a new array and leave the original array unchanged. - * - * @param v Array that should be normalized. - * @return Normalized array. - */ - public static float[] normalizeL2(float[] v) { - double norm = normL2(v); - if (norm > 0.0f) { - float[] vn = new float[v.length]; - for (int i = 0; i < v.length; i++) { - vn[i] = (float) (v[i] / norm); - } - return vn; - } else { - return v; - } - } - - /** - * Normalizes a float array with respect to the L2 (Euclidian) norm. The - * method will perform the normalisation in place. - * - * @param v Array that should be normalized. - * @return Normalized array. - */ - public static float[] normalizeL2InPlace(float[] v) { - double norm = normL2(v); - if (norm > 0.0f) { - for (int i = 0; i < v.length; i++) { - v[i] = (float) (v[i] / norm); - } - return v; - } else { - return v; - } - } - - /** - * Normalizes a double array with respect to the L2 (euclidian) norm. The - * method will return a new array and leave the original array unchanged. - * - * @param v Array that should be normalized. - * @return Normalized array. - */ - public static double[] normalizeL2(double[] v) { - double norm = normL2(v); - if (norm > 0.0f) { - double[] vn = new double[v.length]; - for (int i = 0; i < v.length; i++) { - vn[i] = (float) (v[i] / norm); - } - return vn; - } else { - return v; - } - } - - - /** - * Calculates and returns the L2 (euclidian) norm of a float array. - * - * @param v Float array for which to calculate the L2 norm. - * @return L2 norm - */ - public static double normL2(float[] v) { - float dist = 0; - for(int i = 0; i < v.length; i++){ - dist += Math.pow(v[i], 2); - } - return Math.sqrt(dist); - } - - /** - * Calculates and returns the L2 (euclidian) norm of a double array. - * - * @param v Double array for which to calculate the L2 norm. - * @return L2 norm - */ - public static double normL2(double[] v) { - float dist = 0; - for(int i = 0; i < v.length; i++){ - dist += Math.pow(v[i], 2); - } - return Math.sqrt(dist); - } - - - /** - * Checks whether the provided array is a zero array or not. - * - * @param array Array to check - * @return true if array is not zero, false otherwise. - */ - public static boolean checkNotZero(double[] array) { - for (double v : array) { - if (v > 0.0 || v < 0.0) { + private MathHelper() { + } + + public static final double SQRT2 = Math.sqrt(2); + public static final float SQRT2_f = (float) SQRT2; + + public static double getScore(double dist, double maxDist) { + if (Double.isNaN(dist) || Double.isNaN(maxDist)) { + return 0d; + } + double score = 1d - (dist / maxDist); + if (score > 1d) { + return 1d; + } + if (score < 0d) { + return 0d; + } + return score; + } + + /** + * Normalizes a float array with respect to the L2 (euclidian) norm. The method will return a new array and leave the original array unchanged. + * + * @param v Array that should be normalized. + * @return Normalized array. + */ + public static float[] normalizeL2(float[] v) { + double norm = normL2(v); + if (norm > 0.0f) { + float[] vn = new float[v.length]; + for (int i = 0; i < v.length; i++) { + vn[i] = (float) (v[i] / norm); + } + return vn; + } else { + return v; + } + } + + /** + * Normalizes a float array with respect to the L2 (Euclidian) norm. The method will perform the normalisation in place. + * + * @param v Array that should be normalized. + * @return Normalized array. + */ + public static float[] normalizeL2InPlace(float[] v) { + double norm = normL2(v); + if (norm > 0.0f) { + for (int i = 0; i < v.length; i++) { + v[i] = (float) (v[i] / norm); + } + return v; + } else { + return v; + } + } + + /** + * Normalizes a double array with respect to the L2 (euclidian) norm. The method will return a new array and leave the original array unchanged. + * + * @param v Array that should be normalized. + * @return Normalized array. + */ + public static double[] normalizeL2(double[] v) { + double norm = normL2(v); + if (norm > 0.0f) { + double[] vn = new double[v.length]; + for (int i = 0; i < v.length; i++) { + vn[i] = (float) (v[i] / norm); + } + return vn; + } else { + return v; + } + } + + + /** + * Calculates and returns the L2 (euclidian) norm of a float array. + * + * @param v Float array for which to calculate the L2 norm. + * @return L2 norm + */ + public static double normL2(float[] v) { + float dist = 0; + for (int i = 0; i < v.length; i++) { + dist += Math.pow(v[i], 2); + } + return Math.sqrt(dist); + } + + /** + * Calculates and returns the L2 (euclidian) norm of a double array. + * + * @param v Double array for which to calculate the L2 norm. + * @return L2 norm + */ + public static double normL2(double[] v) { + float dist = 0; + for (int i = 0; i < v.length; i++) { + dist += Math.pow(v[i], 2); + } + return Math.sqrt(dist); + } + + + /** + * Checks whether the provided array is a zero array or not. + * + * @param array Array to check + * @return true if array is not zero, false otherwise. + */ + public static boolean checkNotZero(double[] array) { + for (double v : array) { + if (v > 0.0 || v < 0.0) { return true; } - } - return false; - } - - /** - * Checks whether the provided array is a zero array or not. - * - * @param array Array to check - * @return true if array is not zero, false otherwise. - */ - public static boolean checkNotZero(float[] array) { - for (float v : array) { - if (v > 0.0 || v < 0.0) { + } + return false; + } + + /** + * Checks whether the provided array is a zero array or not. + * + * @param array Array to check + * @return true if array is not zero, false otherwise. + */ + public static boolean checkNotZero(float[] array) { + for (float v : array) { + if (v > 0.0 || v < 0.0) { return true; } - } - return false; - } - - /** - * Checks whether the provided array is a zero array or not. - * - * @param array Array to check - * @return true if array is not zero, false otherwise. - */ - public static boolean checkNotNaN(double[] array) { - for (double v : array) { - if (Double.isNaN(v)) { + } + return false; + } + + /** + * Checks whether the provided array is a zero array or not. + * + * @param array Array to check + * @return true if array is not zero, false otherwise. + */ + public static boolean checkNotNaN(double[] array) { + for (double v : array) { + if (Double.isNaN(v)) { return false; } - } - return true; - } - - /** - * Checks whether the provided array is a zero array or not. - * - * @param array Array to check - * @return true if array is not zero, false otherwise. - */ - public static boolean checkNotNaN(float[] array) { - for (float v : array) { - if (Float.isNaN(v)) { + } + return true; + } + + /** + * Checks whether the provided array is a zero array or not. + * + * @param array Array to check + * @return true if array is not zero, false otherwise. + */ + public static boolean checkNotNaN(float[] array) { + for (float v : array) { + if (Float.isNaN(v)) { return false; } - } - return true; - } - - public static float limit(float val, float min, float max){ - return val > max ? max : (val < min ? min : val); - } - - public static double limit(double val, double min, double max){ - return val > max ? max : (val < min ? min : val); - } - - /** - * Kronecker-Delta function. Returns 1 if i=j and 0 otherwise. - * - * @param i Value of i - * @param j Value of j - * @return Result of Kronecker-Delta. - */ - public static int kronecker(int i, int j) { - if (j == i) { - return 1; - } else { - return 0; - } - } - - public static double[] softmax(double[] input){ - - double[] params = Arrays.copyOf(input, input.length); - - double sum = 0; - - for (int i=0; i d * tolerance); - - return new int[]{(int) h1, (int) k1}; - } - - public static float sum(float[] arr) { - float _result = 0; - for (int i = 0; i < arr.length; i++) { - _result += arr[i]; - } - return _result; - } + } + return true; + } + + public static float limit(float val, float min, float max) { + return val > max ? max : (val < min ? min : val); + } + + public static double limit(double val, double min, double max) { + return val > max ? max : (val < min ? min : val); + } + + /** + * Kronecker-Delta function. Returns 1 if i=j and 0 otherwise. + * + * @param i Value of i + * @param j Value of j + * @return Result of Kronecker-Delta. + */ + public static int kronecker(int i, int j) { + if (j == i) { + return 1; + } else { + return 0; + } + } + + public static double[] softmax(double[] input) { + + double[] params = Arrays.copyOf(input, input.length); + + double sum = 0; + + for (int i = 0; i < params.length; i++) { + params[i] = Math.exp(params[i]); + sum += params[i]; + } + + if (Double.isNaN(sum) || sum < 0) { + Arrays.fill(params, 1.0 / params.length); + } else { + for (int i = 0; i < params.length; i++) { + params[i] = params[i] / sum; + } + } + + return params; + + } + + public static ArgMaxResult argMax(double[] params) { + int maxIndex = 0; + for (int i = 0; i < params.length; i++) { + if (params[maxIndex] < params[i]) { + maxIndex = i; + } + } + return new ArgMaxResult(maxIndex, params[maxIndex]); + } + + + public static class ArgMaxResult { + + private int index; + private double maxValue; + + public ArgMaxResult(int index, double maxValue) { + this.index = index; + this.maxValue = maxValue; + } + + public int getIndex() { + return index; + } + + public double getMaxValue() { + return maxValue; + } + } + + //based on https://jonisalonen.com/2012/converting-decimal-numbers-to-ratios/ + public static int[] toFraction(double d) { + if (d < 0) { + int[] fraction = toFraction(-d); + fraction[0] *= -1; + return fraction; + } + + final double tolerance = 1.0E-6; + double h1 = 1d, h2 = 0d; + double k1 = 0d, k2 = 1d; + double b = d; + do { + double a = Math.floor(b); + double aux = h1; + h1 = a * h1 + h2; + h2 = aux; + aux = k1; + k1 = a * k1 + k2; + k2 = aux; + b = 1d / (b - a); + } while (Math.abs(d - h1 / k1) > d * tolerance); + + return new int[]{(int) h1, (int) k1}; + } + + public static float sum(float[] arr) { + float _result = 0; + for (int i = 0; i < arr.length; i++) { + _result += arr[i]; + } + return _result; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MetadataUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MetadataUtil.java index 75ba42334..d33d2a80e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MetadataUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MetadataUtil.java @@ -5,17 +5,18 @@ import com.drew.metadata.Directory; import com.drew.metadata.Metadata; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; public final class MetadataUtil { - private MetadataUtil() {} + + private MetadataUtil() { + } private static final String JSON_EXTENSION = "json"; @@ -23,19 +24,15 @@ private MetadataUtil() {} private static final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); /** - * Reads the {@link Metadata} from the given {@link Path} and returns the first {@link Directory} - * of the specified type, if present. + * Reads the {@link Metadata} from the given {@link Path} and returns the first {@link Directory} of the specified type, if present. * *

    Note that this is an utility method when one is interested in only one specific - * {@code Directory}. Use {@code Metadata} and its factory methods - * (e.g. {@link ImageMetadataReader#readMetadata} if multiple directories or more fine-graded - * control is needed. + * {@code Directory}. Use {@code Metadata} and its factory methods (e.g. {@link ImageMetadataReader#readMetadata} if multiple directories or more fine-graded control is needed. * - * @param path a path from which the directory may be read. + * @param path a path from which the directory may be read. * @param directoryType the {@code Directory} type - * @param the {@code Directory} type - * @return an {@link Optional} containing the first {@code Directory} of type {@code T} of the - * metadata of the file, if present, otherwise an empty {@code Optional}. + * @param the {@code Directory} type + * @return an {@link Optional} containing the first {@code Directory} of type {@code T} of the metadata of the file, if present, otherwise an empty {@code Optional}. */ public static T getMetadataDirectoryOfType(Path path, Class directoryType) { @@ -46,20 +43,17 @@ public static T getMetadataDirectoryOfType(Path path, logger.error("Error while reading exif data of file {}: {}", path, LogHelper.getStackTrace(e)); } - if(metadata==null){ + if (metadata == null) { return null; } - return metadata.getFirstDirectoryOfType( directoryType ); + return metadata.getFirstDirectoryOfType(directoryType); } /** - * Extracts the JSON tree structure as a {@link JsonNode} of the accompanying JSON metadata file - * named after the given file. That is, both files have the same name without the respective - * extension, e.g. {@code image_00001.json} for {@code image_00001.jpg}. + * Extracts the JSON tree structure as a {@link JsonNode} of the accompanying JSON metadata file named after the given file. That is, both files have the same name without the respective extension, e.g. {@code image_00001.json} for {@code image_00001.jpg}. * * @param objectPath the path to the original file - * @return an {@link Optional} containing the JSON as a {@code JsonNode}, if available and valid, - * otherwise an empty {@code Optional}. + * @return an {@link Optional} containing the JSON as a {@code JsonNode}, if available and valid, otherwise an empty {@code Optional}. */ public static Optional getJsonMetadata(Path objectPath) { String fileName = objectPath.getFileName().toString(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MimeTypeHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MimeTypeHelper.java index c47dca2f6..297b84fa6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MimeTypeHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MimeTypeHelper.java @@ -6,11 +6,12 @@ public final class MimeTypeHelper { - private MimeTypeHelper(){} + private MimeTypeHelper() { + } - private static final HashMap FILETYPES; + private static final HashMap FILETYPES; - static{ + static { FILETYPES = new HashMap<>(); @@ -60,19 +61,19 @@ private MimeTypeHelper(){} FILETYPES.put("off", "application/3d-off"); } - public static String getContentType(File file){ + public static String getContentType(File file) { return getContentType(file.getName()); } - public static String getContentType(Path file){ + public static String getContentType(Path file) { return getContentType(file.getFileName().toString()); } - public static String getContentType(String filename){ + public static String getContentType(String filename) { final String[] split = filename.split("\\."); if (split.length > 0) { - return FILETYPES.getOrDefault(split[split.length-1].toLowerCase(), "application/octet-stream"); + return FILETYPES.getOrDefault(split[split.length - 1].toLowerCase(), "application/octet-stream"); } else { return "application/octet-stream"; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java index 8288651cb..dfe9523dd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MotionHistoryImage.java @@ -1,98 +1,98 @@ package org.vitrivr.cineast.core.util; +import java.util.ArrayList; import org.vitrivr.cineast.core.color.ReadableRGBContainer; -import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import java.util.ArrayList; - public class MotionHistoryImage { - private MotionHistoryImage(int width, int height){ - this.width = width; - this.height = height; - this.intensities = new byte[width * height]; - } - - private final int width, height; - private final byte[] intensities; - - public int getWidth(){ - return this.width; - } - - public int getHeight(){ - return this.height; - } - - public byte[] getIntensities(){ - return this.intensities; - } - - public static MotionHistoryImage motionHistoryImage(SegmentContainer container, int lifeTime, int threshold){ - return motionHistoryImage(container, lifeTime, threshold, true); - } - /** - * @param lifeTime number of frames to consider for image - * @param threshold threshold distance [0, 255] - * @param useThumbnails produce image based on thumbnails to entire frame - */ - public static MotionHistoryImage motionHistoryImage(SegmentContainer container, int lifeTime, int threshold, boolean useThumbnails){ - if(container.getVideoFrames().isEmpty()){ - return null; - } - - ArrayList images = new ArrayList(container.getVideoFrames().size()); - for(VideoFrame f : container.getVideoFrames()){ - if(useThumbnails){ - images.add(f.getImage().getThumbnailColors()); - }else{ - images.add(f.getImage().getColors()); - } - } - - MultiImage first = container.getVideoFrames().get(0).getImage(); - - MotionHistoryImage _return = new MotionHistoryImage( - useThumbnails ? first.getThumbnailImage().getWidth() : first.getWidth(), - useThumbnails ? first.getThumbnailImage().getHeight() : first.getHeight()); - - if(container.getVideoFrames().size() == 1){ - return _return; - } - - float sub = 1f / lifeTime; - - float[] tmp = new float[images.get(0).length]; - for(int i = 1; i < images.size(); ++i){ - int[] current = images.get(i); - int[] last = images.get(i - 1); - - for(int j = 0; j < current.length; ++j){ - int dist = dist(last[j], current[j]); - if(dist > threshold){ - tmp[j] = 1f; - }else{ - tmp[j] = Math.max(0f, tmp[j] - sub); - } - } - - } - - for(int i = 0; i < tmp.length; ++i){ - _return.intensities[i] = (byte)Math.round(127f * tmp[i]); - } - - return _return; - } - - private static int dist(int color1, int color2){ - - float l1 = 0.2126f * ReadableRGBContainer.getRed(color1) + 0.7152f * ReadableRGBContainer.getGreen(color1) + 0.0722f * ReadableRGBContainer.getBlue(color1); - float l2 = 0.2126f * ReadableRGBContainer.getRed(color2) + 0.7152f * ReadableRGBContainer.getGreen(color2) + 0.0722f * ReadableRGBContainer.getBlue(color2); - - return Math.round(Math.abs(l1 - l2)); - } - + private MotionHistoryImage(int width, int height) { + this.width = width; + this.height = height; + this.intensities = new byte[width * height]; + } + + private final int width, height; + private final byte[] intensities; + + public int getWidth() { + return this.width; + } + + public int getHeight() { + return this.height; + } + + public byte[] getIntensities() { + return this.intensities; + } + + public static MotionHistoryImage motionHistoryImage(SegmentContainer container, int lifeTime, int threshold) { + return motionHistoryImage(container, lifeTime, threshold, true); + } + + /** + * @param lifeTime number of frames to consider for image + * @param threshold threshold distance [0, 255] + * @param useThumbnails produce image based on thumbnails to entire frame + */ + public static MotionHistoryImage motionHistoryImage(SegmentContainer container, int lifeTime, int threshold, boolean useThumbnails) { + if (container.getVideoFrames().isEmpty()) { + return null; + } + + ArrayList images = new ArrayList(container.getVideoFrames().size()); + for (VideoFrame f : container.getVideoFrames()) { + if (useThumbnails) { + images.add(f.getImage().getThumbnailColors()); + } else { + images.add(f.getImage().getColors()); + } + } + + MultiImage first = container.getVideoFrames().get(0).getImage(); + + MotionHistoryImage _return = new MotionHistoryImage( + useThumbnails ? first.getThumbnailImage().getWidth() : first.getWidth(), + useThumbnails ? first.getThumbnailImage().getHeight() : first.getHeight()); + + if (container.getVideoFrames().size() == 1) { + return _return; + } + + float sub = 1f / lifeTime; + + float[] tmp = new float[images.get(0).length]; + for (int i = 1; i < images.size(); ++i) { + int[] current = images.get(i); + int[] last = images.get(i - 1); + + for (int j = 0; j < current.length; ++j) { + int dist = dist(last[j], current[j]); + if (dist > threshold) { + tmp[j] = 1f; + } else { + tmp[j] = Math.max(0f, tmp[j] - sub); + } + } + + } + + for (int i = 0; i < tmp.length; ++i) { + _return.intensities[i] = (byte) Math.round(127f * tmp[i]); + } + + return _return; + } + + private static int dist(int color1, int color2) { + + float l1 = 0.2126f * ReadableRGBContainer.getRed(color1) + 0.7152f * ReadableRGBContainer.getGreen(color1) + 0.0722f * ReadableRGBContainer.getBlue(color1); + float l2 = 0.2126f * ReadableRGBContainer.getRed(color2) + 0.7152f * ReadableRGBContainer.getGreen(color2) + 0.0722f * ReadableRGBContainer.getBlue(color2); + + return Math.round(Math.abs(l1 - l2)); + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MultiTracker.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MultiTracker.java index 1cba463f1..69162d47d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MultiTracker.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/MultiTracker.java @@ -4,15 +4,12 @@ import boofcv.factory.tracker.FactoryTrackerObjectQuad; import boofcv.struct.image.GrayU8; import georegression.struct.shapes.Quadrilateral_F64; -import org.vitrivr.cineast.core.data.Pair; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.data.Pair; /** - * MultiTracker allows for multiple objects to be tracked across frames. - * 1. Initialize the MultiTracker with the initial frame and coordinates of the objects to be tracked - * 2. Call update(GrayU8 frame) with the new frame to obtain the coordinates of the objects in the new frame + * MultiTracker allows for multiple objects to be tracked across frames. 1. Initialize the MultiTracker with the initial frame and coordinates of the objects to be tracked 2. Call update(GrayU8 frame) with the new frame to obtain the coordinates of the objects in the new frame */ public class MultiTracker { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java index a5bc32dfe..e06ee2bfb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/OptionalUtil.java @@ -1,29 +1,25 @@ package org.vitrivr.cineast.core.util; -import org.vitrivr.cineast.core.data.Pair; - import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.vitrivr.cineast.core.data.Pair; public final class OptionalUtil { - private OptionalUtil() {} + + private OptionalUtil() { + } /** - * If both a value of the first {@code Optional} and a value produced by the supplying function is - * present, returns a {@code Pair} of both, otherwise returns an empty {@code Optional}. + * If both a value of the first {@code Optional} and a value produced by the supplying function is present, returns a {@code Pair} of both, otherwise returns an empty {@code Optional}. * - * @param first the first {@code Optional} - * @param secondSupplier the supplying function that produces a second {@code Optional} to be - * returned - * @return an {@code Optional} describing the value of the first {@code Optional} and the result - * produced by the supplying function, if both are present, otherwise an empty - * {@code Optional}. - * @throws NullPointerException if the first {@code Optional} is null or the supplying function or - * its result are {@code null} + * @param first the first {@code Optional} + * @param secondSupplier the supplying function that produces a second {@code Optional} to be returned + * @return an {@code Optional} describing the value of the first {@code Optional} and the result produced by the supplying function, if both are present, otherwise an empty {@code Optional}. + * @throws NullPointerException if the first {@code Optional} is null or the supplying function or its result are {@code null} */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static Optional> and(Optional first, @@ -31,20 +27,16 @@ public static Optional> and(Optional first, Objects.requireNonNull(first); Objects.requireNonNull(secondSupplier); return first.flatMap(t -> Objects.requireNonNull(secondSupplier.get()) - .map(u -> new Pair<>(t, u))); + .map(u -> new Pair<>(t, u))); } /** - * If a value of the first {@code Optional} is present, returns it, otherwise returns an - * {@code Optional} produced by the supplying function. + * If a value of the first {@code Optional} is present, returns it, otherwise returns an {@code Optional} produced by the supplying function. * - * @param first the first {@link Optional} - * @param secondSupplier the supplying function that produces a second {@link Optional} to be - * returned - * @return an {@link Optional} describing the value of the first {@link Optional}, if present, - * otherwise an {@link Optional} produced by the supplying function. - * @throws NullPointerException if the first {@link Optional} is null or the supplying function or - * its result is null + * @param first the first {@link Optional} + * @param secondSupplier the supplying function that produces a second {@link Optional} to be returned + * @return an {@link Optional} describing the value of the first {@link Optional}, if present, otherwise an {@link Optional} produced by the supplying function. + * @throws NullPointerException if the first {@link Optional} is null or the supplying function or its result is null */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static Optional or(Optional first, Supplier> secondSupplier) { @@ -58,8 +50,7 @@ public static Optional or(Optional first, Supplier> second } /** - * If a value of the given {@code Optional} is present, returns a sequential {@link Stream} - * containing only that value, otherwise returns an empty {@code Stream}. + * If a value of the given {@code Optional} is present, returns a sequential {@link Stream} containing only that value, otherwise returns an empty {@code Stream}. * * @param optional the {@code Optional} * @return the optional value as a {@code Stream} @@ -74,8 +65,7 @@ public static Stream toStream(Optional optional) { } /** - * If a value of the given {@code Optional} is present, returns a {@link List} containing only - * that value, otherwise returns an empty {@code List}. + * If a value of the given {@code Optional} is present, returns a {@link List} containing only that value, otherwise returns an empty {@code List}. * * @param optional the {@code Optional} * @return the optional value as a {@code List} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/QueryIDGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/QueryIDGenerator.java index e8e6f34cb..94b639f66 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/QueryIDGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/QueryIDGenerator.java @@ -5,6 +5,6 @@ public class QueryIDGenerator { public static String generateQueryID() { - return "q-"+RandomStringUtils.randomNumeric(3); + return "q-" + RandomStringUtils.randomNumeric(3); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/RandomStringGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/RandomStringGenerator.java index 1097c3678..ce5f2f8bd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/RandomStringGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/RandomStringGenerator.java @@ -1,9 +1,9 @@ package org.vitrivr.cineast.core.util; import java.util.Random; + /** * Generates random strings to be used for Ids - * */ public class RandomStringGenerator { @@ -11,10 +11,10 @@ private RandomStringGenerator() { } public static final int DEFAULT_LENGTH = 16; - public static final char[] DEFAULT_ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + public static final char[] DEFAULT_ALPHABET = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', - 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' }; + 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'}; private static final Random r = new Random(); @@ -33,20 +33,21 @@ public static String generateRandomString(int length) { public static String generateRandomString(String prefix, int length) { return generateRandomString(prefix, length, DEFAULT_ALPHABET); } - - public static String generateRandomString(String prefix){ + + public static String generateRandomString(String prefix) { return generateRandomString(prefix, DEFAULT_LENGTH); } - - public static String generateRandomString(){ + + public static String generateRandomString() { return generateRandomString("", DEFAULT_LENGTH, DEFAULT_ALPHABET); } /** - * Generates a random string from a given alphabet with a given length. The prefix of the string can be optionally specified. - * @param random the random number generator to be used for string generation - * @param prefix the prefix for the generated string. if null is provided, the string will have no prefix. - * @param length the length of the string to generate, must be positive + * Generates a random string from a given alphabet with a given length. The prefix of the string can be optionally specified. + * + * @param random the random number generator to be used for string generation + * @param prefix the prefix for the generated string. if null is provided, the string will have no prefix. + * @param length the length of the string to generate, must be positive * @param alphabet the set of characters from which to generate the string */ public static String generateRandomString(Random random, String prefix, int length, diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java index 60a76df58..4d1e54adb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.util; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.MediaType; @@ -7,309 +10,303 @@ import org.vitrivr.cineast.core.extraction.decode.general.Converter; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; +import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.features.codebook.CodebookGenerator; import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.features.retriever.Retriever; -import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; public class ReflectionHelper { - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); - /** Name of the package where {@link Extractor} and {@link Retriever} classes are located by default. */ - public static final String FEATURE_MODULE_PACKAGE = "org.vitrivr.cineast.core.features"; + /** + * Name of the package where {@link Extractor} and {@link Retriever} classes are located by default. + */ + public static final String FEATURE_MODULE_PACKAGE = "org.vitrivr.cineast.core.features"; - /** Name of the package where {@link CodebookGenerator} classes are located by default. */ - private static final String CODEBOOK_GENERATOR_PACKAGE = "org.vitrivr.cineast.core.features.codebook"; + /** + * Name of the package where {@link CodebookGenerator} classes are located by default. + */ + private static final String CODEBOOK_GENERATOR_PACKAGE = "org.vitrivr.cineast.core.features.codebook"; - /** Name of the package where Exporter classes are located by default. */ - private static final String EXPORTER_PACKAGE = "org.vitrivr.cineast.core.features.exporter"; + /** + * Name of the package where Exporter classes are located by default. + */ + private static final String EXPORTER_PACKAGE = "org.vitrivr.cineast.core.features.exporter"; - /** Name of the package where {@link ObjectIdGenerator} classes are located by default. */ - private static final String IDGENERATOR_PACKAGE = "org.vitrivr.cineast.core.extraction.idgenerator"; + /** + * Name of the package where {@link ObjectIdGenerator} classes are located by default. + */ + private static final String IDGENERATOR_PACKAGE = "org.vitrivr.cineast.core.extraction.idgenerator"; - /** Name of the package where {@link MetadataExtractor} classes are located by default. */ - private static final String METADATA_PACKAGE = "org.vitrivr.cineast.core.extraction.metadata"; + /** + * Name of the package where {@link MetadataExtractor} classes are located by default. + */ + private static final String METADATA_PACKAGE = "org.vitrivr.cineast.core.extraction.metadata"; - /** Name of the package where {@link Decoder} classes are located by default. */ - private static final String DECODER_PACKAGE = "org.vitrivr.cineast.core.extraction.decode"; + /** + * Name of the package where {@link Decoder} classes are located by default. + */ + private static final String DECODER_PACKAGE = "org.vitrivr.cineast.core.extraction.decode"; - /** - * Tries to instantiate a new, named ObjectIdGenerator object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the IDGENERATOR_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the {@link ObjectIdGenerator}. - * @param properties Properties that should be used to configure new {@link ObjectIdGenerator} - * @return Instance of ObjectIdGenerator or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static ObjectIdGenerator newIdGenerator(String name, Map properties) { - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, ObjectIdGenerator.class, IDGENERATOR_PACKAGE); - } + /** + * Tries to instantiate a new, named ObjectIdGenerator object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the IDGENERATOR_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the {@link ObjectIdGenerator}. + * @param properties Properties that should be used to configure new {@link ObjectIdGenerator} + * @return Instance of ObjectIdGenerator or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static ObjectIdGenerator newIdGenerator(String name, Map properties) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, ObjectIdGenerator.class, IDGENERATOR_PACKAGE); + } - if (properties == null || properties.isEmpty()) { - return instantiate(c); - } else { - return instantiate(c, new Class[]{Map.class}, properties); - } - } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create ObjectIdGenerator. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + if (properties == null || properties.isEmpty()) { + return instantiate(c); + } else { + return instantiate(c, new Class[]{Map.class}, properties); + } + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create ObjectIdGenerator. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named CodebookGenerator object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the CODEBOOK_GENERATOR_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the CodebookGenerator. - * @return Instance of CodebookGenerator or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static CodebookGenerator newCodebookGenerator(String name) { - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, CodebookGenerator.class, CODEBOOK_GENERATOR_PACKAGE); - } - return instantiate(c); - } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create CodebookGenerator. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + /** + * Tries to instantiate a new, named CodebookGenerator object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the CODEBOOK_GENERATOR_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the CodebookGenerator. + * @return Instance of CodebookGenerator or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static CodebookGenerator newCodebookGenerator(String name) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, CodebookGenerator.class, CODEBOOK_GENERATOR_PACKAGE); + } + return instantiate(c); + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create CodebookGenerator. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named Covnerter object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * @param fqn The fully-qualified name of the converter. - * @return Instance of Converter or null, if instantiation fails. - */ - public static Converter newConverter(String fqn) { - try { - Class c = Class.forName(fqn); - if(!c.isAssignableFrom(Converter.class)){ - LOGGER.fatal("Failed to create Converter. Class '{}' is not a converter", fqn); - return null; - } - @SuppressWarnings("unchecked") + /** + * Tries to instantiate a new, named Covnerter object. If the methods succeeds to do so, that instance is returned by the method. + * + * @param fqn The fully-qualified name of the converter. + * @return Instance of Converter or null, if instantiation fails. + */ + public static Converter newConverter(String fqn) { + try { + Class c = Class.forName(fqn); + if (!c.isAssignableFrom(Converter.class)) { + LOGGER.fatal("Failed to create Converter. Class '{}' is not a converter", fqn); + return null; + } + @SuppressWarnings("unchecked") Class cc = (Class) c; - return instantiate(cc); - } catch (ClassNotFoundException e) { - LOGGER.fatal("Failed to create Converter. Could not find or access class with name {} ({}).", fqn, LogHelper.getStackTrace(e)); - return null; - } - } + return instantiate(cc); + } catch (ClassNotFoundException e) { + LOGGER.fatal("Failed to create Converter. Could not find or access class with name {} ({}).", fqn, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named Extractor object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the FEATURE_MODULE_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the Exporter. - * @return Instance of Exporter or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static Extractor newExtractor(String name) { - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, Extractor.class, FEATURE_MODULE_PACKAGE); - } - return instantiate(c); - } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create Exporter. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + /** + * Tries to instantiate a new, named Extractor object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the FEATURE_MODULE_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the Exporter. + * @return Instance of Exporter or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static Extractor newExtractor(String name) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, Extractor.class, FEATURE_MODULE_PACKAGE); + } + return instantiate(c); + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create Exporter. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named Exporter object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the EXPORTER_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the Exporter. - * @return Instance of Exporter or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static Extractor newExporter(String name, Map configuration) { - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, Extractor.class, EXPORTER_PACKAGE); - } + /** + * Tries to instantiate a new, named Exporter object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the EXPORTER_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the Exporter. + * @return Instance of Exporter or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static Extractor newExporter(String name, Map configuration) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, Extractor.class, EXPORTER_PACKAGE); + } - if (configuration == null || configuration.isEmpty()) { - return instantiate(c); - } else { - return instantiate(c, configuration); - } - } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create Exporter. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + if (configuration == null || configuration.isEmpty()) { + return instantiate(c); + } else { + return instantiate(c, configuration); + } + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create Exporter. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named {@link Decoder} object. If the methods succeeds to do so, that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the {@link #DECODER_PACKAGE} - * is assumed together with the {@link MediaType#getName()} and the name is treated as simple name. - * - * @param name Name of the Decoder. - * @return Instance of Decoder or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static Decoder newDecoder(String name, MediaType type){ - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, Decoder.class, DECODER_PACKAGE+"."+type.getName()); - } - return instantiate(c); - } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create Decoder. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + /** + * Tries to instantiate a new, named {@link Decoder} object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the {@link #DECODER_PACKAGE} is assumed together with the {@link MediaType#getName()} and the name is treated as simple name. + * + * @param name Name of the Decoder. + * @return Instance of Decoder or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static Decoder newDecoder(String name, MediaType type) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, Decoder.class, DECODER_PACKAGE + "." + type.getName()); + } + return instantiate(c); + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create Decoder. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named MetadataExtractor object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the METADATA_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the MetadataExtractor. - * @return Instance of MetadataExtractor or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static MetadataExtractor newMetadataExtractor(String name) { - Class c = null; - try { - if (name.contains(".")) { - c = (Class) Class.forName(name); - } else { - c = getClassFromName(name, MetadataExtractor.class, METADATA_PACKAGE); - } + /** + * Tries to instantiate a new, named MetadataExtractor object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the METADATA_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the MetadataExtractor. + * @return Instance of MetadataExtractor or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static MetadataExtractor newMetadataExtractor(String name) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, MetadataExtractor.class, METADATA_PACKAGE); + } - return instantiate(c); - } catch (ClassNotFoundException | InstantiationException | ClassCastException e) { - LOGGER.fatal("Failed to create MetadataExtractor. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + return instantiate(c); + } catch (ClassNotFoundException | InstantiationException | ClassCastException e) { + LOGGER.fatal("Failed to create MetadataExtractor. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - /** - * Tries to instantiate a new, named MetadataExtractor object. If the methods succeeds to do so, - * that instance is returned by the method. - * - * If the name contains dots (.), that name is treated as FQN. Otherwise, the METADATA_PACKAGE - * is assumed and the name is treated as simple name. - * - * @param name Name of the MetadataExtractor. - * @return Instance of MetadataExtractor or null, if instantiation fails. - */ - @SuppressWarnings("unchecked") - public static Segmenter newSegmenter(String name, Map configuration, ExtractionContextProvider provider) { - Class> c = null; - try { - c = (Class>) Class.forName(name); - if (configuration == null) { - return instantiate(c, new Class[]{ExtractionContextProvider.class}, provider); - } else { - return instantiate(c, new Class[]{ExtractionContextProvider.class, Map.class}, provider, configuration); - } - } catch (ClassNotFoundException | ClassCastException e) { - LOGGER.fatal("Failed to create Segmenter. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); - return null; - } - } + /** + * Tries to instantiate a new, named MetadataExtractor object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the METADATA_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the MetadataExtractor. + * @return Instance of MetadataExtractor or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static Segmenter newSegmenter(String name, Map configuration, ExtractionContextProvider provider) { + Class> c = null; + try { + c = (Class>) Class.forName(name); + if (configuration == null) { + return instantiate(c, new Class[]{ExtractionContextProvider.class}, provider); + } else { + return instantiate(c, new Class[]{ExtractionContextProvider.class, Map.class}, provider, configuration); + } + } catch (ClassNotFoundException | ClassCastException e) { + LOGGER.fatal("Failed to create Segmenter. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } - private static Class[] getClassArray(Object... args) { - Class[] cls = new Class[args.length]; - int i = 0; - for (Object o : args) { - cls[i++] = o.getClass(); - } - return cls; - } + private static Class[] getClassArray(Object... args) { + Class[] cls = new Class[args.length]; + int i = 0; + for (Object o : args) { + cls[i++] = o.getClass(); + } + return cls; + } - /** - * Convenience method to instantiate an object of a given class using a specific constructor. - * - * @param cl The class that should be instantiated. - * @param args The arguments that should be passed to the constructor. The constructor signature will be inferred from this list. - * @return Instance of the class or null, if instantiation failed. - */ - public static T instantiate(Class cl, Object... args) { - return instantiate(cl, getClassArray(args), args); - } + /** + * Convenience method to instantiate an object of a given class using a specific constructor. + * + * @param cl The class that should be instantiated. + * @param args The arguments that should be passed to the constructor. The constructor signature will be inferred from this list. + * @return Instance of the class or null, if instantiation failed. + */ + public static T instantiate(Class cl, Object... args) { + return instantiate(cl, getClassArray(args), args); + } - /** - * Convenience method to instantiate an object of a given class using a defined constructor. - * - * @param cl The class that should be instantiated. - * @param types An array of types that defines the expected signature of the class's constructor. - * @param args The arguments that should be passed to the constructor. - * @return Instance of the class or null, if instantiation failed. - */ - public static T instantiate(Class cl, Class[] types, Object... args) { - try { - Constructor con = cl.getConstructor(types); - return con.newInstance(args); - } catch (InvocationTargetException e) { - LOGGER.error("InvocationTargetException: {}", LogHelper.getStackTrace(e.getCause())); - } catch (Exception e) { - LOGGER.error(LogHelper.getStackTrace(e)); - } - return null; - } + /** + * Convenience method to instantiate an object of a given class using a defined constructor. + * + * @param cl The class that should be instantiated. + * @param types An array of types that defines the expected signature of the class's constructor. + * @param args The arguments that should be passed to the constructor. + * @return Instance of the class or null, if instantiation failed. + */ + public static T instantiate(Class cl, Class[] types, Object... args) { + try { + Constructor con = cl.getConstructor(types); + return con.newInstance(args); + } catch (InvocationTargetException e) { + LOGGER.error("InvocationTargetException: {}", LogHelper.getStackTrace(e.getCause())); + } catch (Exception e) { + LOGGER.error(LogHelper.getStackTrace(e)); + } + return null; + } - @SuppressWarnings("unchecked") - public static Class getClassFromName(String className, Class expectedSuperClass, String expectedPackage) throws IllegalArgumentException, ClassNotFoundException, InstantiationException { - Class targetClass = null; - try { - String classPath = expectedPackage + "." + className; - Class c = Class.forName(classPath); - if(!expectedSuperClass.isAssignableFrom(c)){ - throw new InstantiationException(classPath + " is not a sub-class of " + expectedSuperClass.getName()); - } - targetClass = (Class) c; - } catch(UnsupportedOperationException e){ - LOGGER.warn("'name' was not a string during class instantiation in instantiateFromJson"); - } - return targetClass; - } + @SuppressWarnings("unchecked") + public static Class getClassFromName(String className, Class expectedSuperClass, String expectedPackage) throws IllegalArgumentException, ClassNotFoundException, InstantiationException { + Class targetClass = null; + try { + String classPath = expectedPackage + "." + className; + Class c = Class.forName(classPath); + if (!expectedSuperClass.isAssignableFrom(c)) { + throw new InstantiationException(classPath + " is not a sub-class of " + expectedSuperClass.getName()); + } + targetClass = (Class) c; + } catch (UnsupportedOperationException e) { + LOGGER.warn("'name' was not a string during class instantiation in instantiateFromJson"); + } + return targetClass; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ScoreFusion.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ScoreFusion.java index b8fad9884..47b8b128b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ScoreFusion.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ScoreFusion.java @@ -2,34 +2,31 @@ import com.google.common.collect.ListMultimap; import gnu.trove.map.TObjectDoubleMap; +import java.util.List; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; import org.vitrivr.cineast.core.db.dao.reader.MediaSegmentReader; -import java.util.List; -import java.util.Set; - public class ScoreFusion { - private ScoreFusion() {} + + private ScoreFusion() { + } private static final Logger logger = LogManager.getLogger(); /** - * Fuses the object scores into the segment scores by adding every object score to the scores of - * its segments. If an object without any of its segments was found, the first segment gets added - * and used instead. - * Note that this method modifies {@code scoreBySegmentId} in place without changing - * {@code scoreByObjectId}. + * Fuses the object scores into the segment scores by adding every object score to the scores of its segments. If an object without any of its segments was found, the first segment gets added and used instead. Note that this method modifies {@code scoreBySegmentId} in place without changing {@code scoreByObjectId}. * * @param scoreBySegmentId segment ids with their respective score - * @param scoreByObjectId object ids with their respective score + * @param scoreByObjectId object ids with their respective score */ public static void fuseObjectsIntoSegments(TObjectDoubleMap scoreBySegmentId, TObjectDoubleMap scoreByObjectId, MediaSegmentReader mediaSegmentReader) { Set objectIds = scoreByObjectId.keySet(); - if(objectIds.isEmpty()){ + if (objectIds.isEmpty()) { return; } ListMultimap segmentsByObjectId = mediaSegmentReader.lookUpSegmentsOfObjects(objectIds); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextDetector_EAST.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextDetector_EAST.java index 310983132..9c8cc4a00 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextDetector_EAST.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TextDetector_EAST.java @@ -1,15 +1,21 @@ package org.vitrivr.cineast.core.util; -import org.opencv.core.*; +import java.util.ArrayList; +import java.util.List; +import org.opencv.core.Mat; +import org.opencv.core.MatOfFloat; +import org.opencv.core.MatOfInt; +import org.opencv.core.MatOfRotatedRect; +import org.opencv.core.Point; +import org.opencv.core.RotatedRect; +import org.opencv.core.Scalar; +import org.opencv.core.Size; import org.opencv.dnn.Dnn; import org.opencv.dnn.Net; import org.opencv.imgproc.Imgproc; import org.opencv.utils.Converters; import org.vitrivr.cineast.core.data.Pair; -import java.util.ArrayList; -import java.util.List; - /** * TextDetector_EAST is able to detect scene text contained within an image */ @@ -134,23 +140,25 @@ private Point[][] detect(Mat scores, Mat geometry, Size size, int rows, int cols } - public List detect (List frames, int batchSize) { + public List detect(List frames, int batchSize) { assert model != null : "Model has not been initialized!"; List allPoints = new ArrayList<>(); - if (frames.size() == 0) { return allPoints; } + if (frames.size() == 0) { + return allPoints; + } frames.forEach(frame -> Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB)); int rows = frames.get(0).rows(); int cols = frames.get(0).cols(); Size size = new Size(frames.get(0).width() - (frames.get(0).width() % 32), frames.get(0).height() - (frames.get(0).height() % 32)); int H = (int) (size.height / 4); - for (int i=0; i subFrames = frames.subList(i, Math.min(i + batchSize, frames.size())); Mat blob = Dnn.blobFromImages(subFrames, 1.0, size, new Scalar(123.68, 116.78, 103.94), true, false); List outs = new ArrayList<>(2); model.setInput(blob); model.forward(outs, outNames); - for (int j=0; j { - private final LoadingCache cache; + private final LoadingCache cache; - public ThreadLocalObjectCache(CacheLoader cacheLoader) { - this.cache = CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES).build(cacheLoader); - } + public ThreadLocalObjectCache(CacheLoader cacheLoader) { + this.cache = CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES).build(cacheLoader); + } - public T get() { - try { - return this.cache.get(Thread.currentThread()); - } catch (ExecutionException executionException) { - throw new IllegalStateException("Error accessing ThreadLocalObjectCache for thread " + Thread.currentThread().getName()); - } + public T get() { + try { + return this.cache.get(Thread.currentThread()); + } catch (ExecutionException executionException) { + throw new IllegalStateException("Error accessing ThreadLocalObjectCache for thread " + Thread.currentThread().getName()); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TimeHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TimeHelper.java index 945c046cf..f123f06f2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TimeHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/TimeHelper.java @@ -1,35 +1,35 @@ package org.vitrivr.cineast.core.util; import gnu.trove.stack.array.TDoubleArrayStack; +import java.util.function.Supplier; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.function.Supplier; - public class TimeHelper { - private TimeHelper(){} - - private static TDoubleArrayStack tic = new TDoubleArrayStack(); - - private static final Logger LOGGER = LogManager.getLogger(); - - public static synchronized void tic(){ - tic.push(System.nanoTime() / 1000000d); - } - - public static synchronized double toc(){ - double d = tic.pop(); - return (System.nanoTime() / 1000000d) - d; - } - - public static O timeCall(Supplier fun, String name, - Level level){ - long start = System.currentTimeMillis(); - O res = fun.get(); - long stop = System.currentTimeMillis(); - LOGGER.log(level, "Finished {} in {} ms", name, stop-start); - return res; - } + private TimeHelper() { + } + + private static TDoubleArrayStack tic = new TDoubleArrayStack(); + + private static final Logger LOGGER = LogManager.getLogger(); + + public static synchronized void tic() { + tic.push(System.nanoTime() / 1000000d); + } + + public static synchronized double toc() { + double d = tic.pop(); + return (System.nanoTime() / 1000000d) - d; + } + + public static O timeCall(Supplier fun, String name, + Level level) { + long start = System.currentTimeMillis(); + O res = fun.get(); + long stop = System.currentTimeMillis(); + LOGGER.log(level, "Finished {} in {} ms", name, stop - start); + return res; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/CENS.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/CENS.java index 01dd8ffe9..57d255e8a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/CENS.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/CENS.java @@ -5,62 +5,59 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.HanningWindow; /** - * This class obtains and returns CENS (Chroma Energy distribution Normalized Statistics) features according to [1]. CENS captures the temporal development of the energy distribution in the different - * chroma or pitch-classes. - * + * This class obtains and returns CENS (Chroma Energy distribution Normalized Statistics) features according to [1]. CENS captures the temporal development of the energy distribution in the different chroma or pitch-classes. + *

    * [1] Mueller, M., Kurth, F., & Clausen, M. (2005). Audio matching via chroma-based statistical features. International Conference on Music Information Retrieval, 288–295. http://doi.org/10.1109/ASPAA.2005.1540223 - * - * */ public class CENS { - /** - * Private constructor; this is a pure utility class - */ - private CENS() {} + /** + * Private constructor; this is a pure utility class + */ + private CENS() { + } - /** - * Calculates and returns CENS features given a Harmonic Pitch Class Profile (HPCP). - * - * @param hpcps HPCP from which to calculate the CENS feature. - * @param w Size of the Hanning Window used in the convolution step. First parameter for the algorithm according to [1] - * @param downsample Ratio for downsampling. Second parameter for the algorithm according to [1] - * - * @return 2D double array of CENS features. - */ - public static double[][] cens(HPCP hpcps, int w, int downsample) { - double[] window = (new HanningWindow()).values(w); - double[][] cens = new double[(hpcps.size() + w)/downsample-1][hpcps.getResolution().bins]; - for (int j = 0; j= 0.4f && value <= 1.0f) { - return 4.0f; - } else if (value >= 0.2f && value < 0.4f) { - return 3.0f; - } else if (value >= 0.1f && value < 0.2f) { - return 2.0f; - } else if (value >= 0.05f && value < 0.1f) { - return 1.0f; - } else { - return 0.0f; - } + /** + * Maps normalized energy values to the logarithmic CENS energy scale. + * + * @param value Energy value that should be mapped. + * @return Value on the CENS scale that corresponds to the provided value. + */ + private static float mapToCens(float value) { + if (value >= 0.4f && value <= 1.0f) { + return 4.0f; + } else if (value >= 0.2f && value < 0.4f) { + return 3.0f; + } else if (value >= 0.1f && value < 0.2f) { + return 2.0f; + } else if (value >= 0.05f && value < 0.1f) { + return 1.0f; + } else { + return 0.0f; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/HPCP.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/HPCP.java index 0d468eabd..d60005657 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/HPCP.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/HPCP.java @@ -1,243 +1,235 @@ package org.vitrivr.cineast.core.util.audio; +import java.util.ArrayList; +import java.util.List; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; import org.vitrivr.cineast.core.util.dsp.midi.MidiUtil; -import java.util.ArrayList; -import java.util.List; - /** - * This class can be used to calculate the Harmonic Pitch Class Profile (HPCP) of a Short-Term Fourier Transform or a single - * Spectrum using the approach outlined by E. Gomez in [1]. - * - * Harmonic Pitch Class Profiles assign the frequencies in a power spectrum to the pitch classes on the equal tempered - * scale. Depending on whether only semitones, 1/2 semitones or 1/3 semitones are considered, the HPCP feature vector - * holds either 12, 24 or 36 entries. - * - * [1] Gómez, E. (2006). Tonal description of polyphonic audio for music content processing. - * INFORMS Journal on Computing, 18(3), 294–304. http://doi.org/10.1287/ijoc.1040.0126 - * - * [2] Kurth F. & Müller M. (2008). Efficient Index-Based Audio Matching. - * IEEE TRANSACTIONS ON AUDIO, SPEECH, AND LANGUAGE PROCESSING + * This class can be used to calculate the Harmonic Pitch Class Profile (HPCP) of a Short-Term Fourier Transform or a single Spectrum using the approach outlined by E. Gomez in [1]. + *

    + * Harmonic Pitch Class Profiles assign the frequencies in a power spectrum to the pitch classes on the equal tempered scale. Depending on whether only semitones, 1/2 semitones or 1/3 semitones are considered, the HPCP feature vector holds either 12, 24 or 36 entries. + *

    + * [1] Gómez, E. (2006). Tonal description of polyphonic audio for music content processing. INFORMS Journal on Computing, 18(3), 294–304. http://doi.org/10.1287/ijoc.1040.0126 + *

    + * [2] Kurth F. & Müller M. (2008). Efficient Index-Based Audio Matching. IEEE TRANSACTIONS ON AUDIO, SPEECH, AND LANGUAGE PROCESSING * * @see STFT * @see Spectrum - * */ public class HPCP { - /** - * Defines the resolution of the HPCP (full semitones, 1/2 semitones or 1/3 semitones). - */ - public enum Resolution { - FULLSEMITONE (12), - HALFSEMITONE (24), - THIRDSEMITON (36); - - public final int bins; - - Resolution(int bins) { - this.bins = bins; - } - } - /** Window-size parameter, which defaults to 4/3 semitones as per [1]. */ - private static final float WINDOW = 4f/3f; + /** + * Defines the resolution of the HPCP (full semitones, 1/2 semitones or 1/3 semitones). + */ + public enum Resolution { + FULLSEMITONE(12), + HALFSEMITONE(24), + THIRDSEMITON(36); - /** Float array holding the HPCP data. */ - private final List hpcp = new ArrayList<>(); + public final int bins; - /** Minimum frequency to consider. Defaults to 100Hz as per [1]. */ - private final float minFrequency; - - /** Maximum frequency to consider. Defaults to 5000Hz as per [1]. */ - private final float maxFrequency; - - /** Resolution of the HPCP. */ - private final Resolution resolution; - - /** - * Calculates the center-frequency of a bin defined by its bin number. The bin numbers - * are defined such that bin 0 corresponds to F_REF (440 Hz). An increase of the bin number - * is equivalent to moving 1, 1/2 or 1/3 of a semitone up on the scale. - * - * @param n Desired bin number - * @return r Resolution to use (1, 1/2, or 1/3) - */ - public static double binToCenterFrequency(int n, Resolution r) { - return MidiUtil.F_REF*Math.pow(2, ((float)n/(float)r.bins)); - } - - /** - * Default constructor. Creates an HPCP container for a full semitone, 12 entries HPCP that consider - * frequencies between 100Hz and 5000Hz. - */ - public HPCP() { - this(Resolution.FULLSEMITONE, 100.0f, 5000.0f); + Resolution(int bins) { + this.bins = bins; } - - /** - * Constructor for a custom HPCP container. - * - * @param resolution Desired resolution in semitones (1, 1/2 or 1/3) - * @param minFrequency Minimum frequency to consider. Frequencies below that limit will be ignored. - * @param maxFrequency Maximum frequency to consider. Frequencies above that limit will be ignored. - */ - public HPCP(Resolution resolution, float minFrequency, float maxFrequency) { - this.resolution = resolution; - this.maxFrequency = maxFrequency; - this.minFrequency = minFrequency; + } + + /** + * Window-size parameter, which defaults to 4/3 semitones as per [1]. + */ + private static final float WINDOW = 4f / 3f; + + /** + * Float array holding the HPCP data. + */ + private final List hpcp = new ArrayList<>(); + + /** + * Minimum frequency to consider. Defaults to 100Hz as per [1]. + */ + private final float minFrequency; + + /** + * Maximum frequency to consider. Defaults to 5000Hz as per [1]. + */ + private final float maxFrequency; + + /** + * Resolution of the HPCP. + */ + private final Resolution resolution; + + /** + * Calculates the center-frequency of a bin defined by its bin number. The bin numbers are defined such that bin 0 corresponds to F_REF (440 Hz). An increase of the bin number is equivalent to moving 1, 1/2 or 1/3 of a semitone up on the scale. + * + * @param n Desired bin number + * @return r Resolution to use (1, 1/2, or 1/3) + */ + public static double binToCenterFrequency(int n, Resolution r) { + return MidiUtil.F_REF * Math.pow(2, ((float) n / (float) r.bins)); + } + + /** + * Default constructor. Creates an HPCP container for a full semitone, 12 entries HPCP that consider frequencies between 100Hz and 5000Hz. + */ + public HPCP() { + this(Resolution.FULLSEMITONE, 100.0f, 5000.0f); + } + + /** + * Constructor for a custom HPCP container. + * + * @param resolution Desired resolution in semitones (1, 1/2 or 1/3) + * @param minFrequency Minimum frequency to consider. Frequencies below that limit will be ignored. + * @param maxFrequency Maximum frequency to consider. Frequencies above that limit will be ignored. + */ + public HPCP(Resolution resolution, float minFrequency, float maxFrequency) { + this.resolution = resolution; + this.maxFrequency = maxFrequency; + this.minFrequency = minFrequency; + } + + /** + * Adds the contribution of a STFT to the HPCP by adding the contributions all the power spectra of that STFT to the HPCP. + * + * @param stft STFT that should contribute to the Harmonic Pitch Class Profile. + */ + public void addContribution(STFT stft) { + for (Spectrum spectrum : stft.getPowerSpectrum()) { + this.addContribution(spectrum); } - - /** - * Adds the contribution of a STFT to the HPCP by adding the contributions all the power spectra - * of that STFT to the HPCP. - * - * @param stft STFT that should contribute to the Harmonic Pitch Class Profile. - */ - public void addContribution(STFT stft) { - for (Spectrum spectrum : stft.getPowerSpectrum()) { - this.addContribution(spectrum); - } - } - - /** - * Adds the contribution of a single spectrum to the HPCP. The spectrum can either be a power - * or magnitude spectrum. - * - * @param spectrum Spectrum that should contribute to the Harmonic Pitch Class Profile. - */ - public void addContribution(Spectrum spectrum) { - /* Prune the PowerSpectrum and the Frequencies to the range that is interesting according to min frequency and max frequency.*/ - Spectrum pruned = spectrum.reduced(this.minFrequency, this.maxFrequency); - - double threshold = 1.0e-8; - List> peaks = pruned.findLocalMaxima(threshold, true); - float[] hpcp = new float[this.resolution.bins]; - - /* For each of the semi-tones (according to resolution), add the contribution of every peak. */ - for (int n=0;n peak : peaks) { - if (pruned.getType() == Spectrum.Type.POWER) { - hpcp[n] += peak.second * this.weight(n, peak.first); - } else if (pruned.getType() == Spectrum.Type.MAGNITUDE) { - hpcp[n] += Math.pow(peak.second,2) * this.weight(n, peak.first); - } - } + } + + /** + * Adds the contribution of a single spectrum to the HPCP. The spectrum can either be a power or magnitude spectrum. + * + * @param spectrum Spectrum that should contribute to the Harmonic Pitch Class Profile. + */ + public void addContribution(Spectrum spectrum) { + /* Prune the PowerSpectrum and the Frequencies to the range that is interesting according to min frequency and max frequency.*/ + Spectrum pruned = spectrum.reduced(this.minFrequency, this.maxFrequency); + + double threshold = 1.0e-8; + List> peaks = pruned.findLocalMaxima(threshold, true); + float[] hpcp = new float[this.resolution.bins]; + + /* For each of the semi-tones (according to resolution), add the contribution of every peak. */ + for (int n = 0; n < this.resolution.bins; n++) { + for (Pair peak : peaks) { + if (pruned.getType() == Spectrum.Type.POWER) { + hpcp[n] += peak.second * this.weight(n, peak.first); + } else if (pruned.getType() == Spectrum.Type.MAGNITUDE) { + hpcp[n] += Math.pow(peak.second, 2) * this.weight(n, peak.first); } - - /* */ - this.hpcp.add(hpcp); + } } - /** - * Returns the raw, un-normalized HPCP float array. - * - * @param idx Zero based index of the time-frame for which to return the HPCP. - * @return Float array containing the HPCP. - */ - public float[] getHpcp(int idx) { - return hpcp.get(idx); + /* */ + this.hpcp.add(hpcp); + } + + /** + * Returns the raw, un-normalized HPCP float array. + * + * @param idx Zero based index of the time-frame for which to return the HPCP. + * @return Float array containing the HPCP. + */ + public float[] getHpcp(int idx) { + return hpcp.get(idx); + } + + /** + * Returns the HPCP vectorwhich has been normalized by the sum of its components as proposed in [2]. + * + * @param idx Zero based index of the time-frame for which to return the HPCP. + * @return float array containing the sum-normalized HPCP for the given index. + */ + public float[] getSumNormalizedHpcp(int idx) { + float[] normHpcp = hpcp.get(idx); + float sum = 0.0f; + for (float aNormHpcp : normHpcp) { + sum += aNormHpcp; } - - /** - * Returns the HPCP vectorwhich has been normalized by the sum of its components as - * proposed in [2]. - * - * @param idx Zero based index of the time-frame for which to return the HPCP. - * @return float array containing the sum-normalized HPCP for the given index. - */ - public float[] getSumNormalizedHpcp(int idx) { - float[] normHpcp = hpcp.get(idx); - float sum = 0.0f; - for (float aNormHpcp : normHpcp) { - sum += aNormHpcp; - } - for (int i = 0; i< normHpcp.length; i++) { - normHpcp[i] /= sum; - } - return normHpcp; + for (int i = 0; i < normHpcp.length; i++) { + normHpcp[i] /= sum; } - - /** - * Returns the HPCP vector which has been normalized by the value of its maximum component - * as proposed in the original paper ([1]). - * - * @param idx Zero based index of the time-frame for which to return the HPCP. - * @return float array containing the max-normalized HPCP for the given index. - */ - public float[] getMaxNormalizedHpcp(int idx) { - float[] normHpcp = hpcp.get(idx); - float max = 0.0f; - for (float aNormHpcp : normHpcp) { - max = Math.max(max,aNormHpcp); - } - for (int i = 0; i< normHpcp.length; i++) { - normHpcp[i] /= max; - } - return normHpcp; + return normHpcp; + } + + /** + * Returns the HPCP vector which has been normalized by the value of its maximum component as proposed in the original paper ([1]). + * + * @param idx Zero based index of the time-frame for which to return the HPCP. + * @return float array containing the max-normalized HPCP for the given index. + */ + public float[] getMaxNormalizedHpcp(int idx) { + float[] normHpcp = hpcp.get(idx); + float max = 0.0f; + for (float aNormHpcp : normHpcp) { + max = Math.max(max, aNormHpcp); } - - /** - * Returns the mean HPCP array (i.e. the HPCP arithmetic mean of all - * HPCP contributions). - * - * @return Float array containing the mean HPCP. - */ - public float[] getMeanHpcp() { - float[] normalized = new float[this.resolution.bins]; - for (float[] hpcp : this.hpcp) { - for (int i=0; i< this.resolution.bins;i++) { - normalized[i] += hpcp[i]/(this.hpcp.size()); - } - } - return normalized; + for (int i = 0; i < normHpcp.length; i++) { + normHpcp[i] /= max; } - - /** - * Returns the size of the HPCP which relates to the number of timepoints. - * - * @return Size of the HPCP. - */ - public int size() { - return this.hpcp.size(); + return normHpcp; + } + + /** + * Returns the mean HPCP array (i.e. the HPCP arithmetic mean of all HPCP contributions). + * + * @return Float array containing the mean HPCP. + */ + public float[] getMeanHpcp() { + float[] normalized = new float[this.resolution.bins]; + for (float[] hpcp : this.hpcp) { + for (int i = 0; i < this.resolution.bins; i++) { + normalized[i] += hpcp[i] / (this.hpcp.size()); + } } - - /** - * Getter for the HPCP resolution which gives an indication about the number of bins per timepoint. - * - * @return Resolution of the HPCP. - */ - public Resolution getResolution() { - return resolution; - } - - /** - * Calculates the contribution of a given frequency (usually from a spectrum) to - * a given bin in the HPCP. - * - * @param n Number of the bin. - * @param frequency Frequency to calculate the contribution for. - * @return Contribution of the frequency to the bin. - */ - private double weight(int n, double frequency) { - double frequency_n = binToCenterFrequency(n, this.resolution); - - - double p = Math.log(frequency/frequency_n)/Math.log(2); - - int m1 = (int) Math.ceil(p) * (-1); - int m2 = (int) Math.floor(p) * (-1); - - - double distance = this.resolution.bins*Math.min(Math.abs(p + m1), Math.abs(p + m2)); - - - if (distance > 0.5 * WINDOW) { - return 0; - } else { - return Math.pow(Math.cos((Math.PI/2) * (distance/(0.5 * WINDOW))),2); - } + return normalized; + } + + /** + * Returns the size of the HPCP which relates to the number of timepoints. + * + * @return Size of the HPCP. + */ + public int size() { + return this.hpcp.size(); + } + + /** + * Getter for the HPCP resolution which gives an indication about the number of bins per timepoint. + * + * @return Resolution of the HPCP. + */ + public Resolution getResolution() { + return resolution; + } + + /** + * Calculates the contribution of a given frequency (usually from a spectrum) to a given bin in the HPCP. + * + * @param n Number of the bin. + * @param frequency Frequency to calculate the contribution for. + * @return Contribution of the frequency to the bin. + */ + private double weight(int n, double frequency) { + double frequency_n = binToCenterFrequency(n, this.resolution); + + double p = Math.log(frequency / frequency_n) / Math.log(2); + + int m1 = (int) Math.ceil(p) * (-1); + int m2 = (int) Math.floor(p) * (-1); + + double distance = this.resolution.bins * Math.min(Math.abs(p + m1), Math.abs(p + m2)); + + if (distance > 0.5 * WINDOW) { + return 0; + } else { + return Math.pow(Math.cos((Math.PI / 2) * (distance / (0.5 * WINDOW))), 2); } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java index 54ba7f616..aa8487118 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/MFCC.java @@ -1,203 +1,199 @@ package org.vitrivr.cineast.core.util.audio; -import org.vitrivr.cineast.core.util.dsp.fft.STFT; -import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; - import java.util.ArrayList; import java.util.List; +import org.vitrivr.cineast.core.util.dsp.fft.STFT; +import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; /** - * This class calculates Mel-frequency cepstral coefficient (MFCC) features from the STFT of an audio signal. - * The code was partially taken from the OrangeCow Volume project. See [1] - * - * [1] S. Pfeiffer, and C. Parker, and T. Vincent. 2005. - * OC Volume: Java speech recognition engine. 2005. [March 8th, 2017]. - * + * This class calculates Mel-frequency cepstral coefficient (MFCC) features from the STFT of an audio signal. The code was partially taken from the OrangeCow Volume project. See [1] + *

    + * [1] S. Pfeiffer, and C. Parker, and T. Vincent. 2005. OC Volume: Java speech recognition engine. 2005. [March 8th, 2017]. */ public class MFCC { - /** Minimum frequency to consider. Defaults to 100Hz as per [1]. */ - private final float minFrequency; - - /** Number of Mel filters to use. Defaults to 23 as per [1]. */ - private final int melfilters; - - /** Array holding the cepstrum coefficients (cepstra). Defaults to 13 as per [1]. */ - private float[] cepstra; - - /** - * Default constructor for MFCC class with 13 cepstra, a Mel-Filter bank of size - * 13 and a minimum frequency of 133.0f Hz. - */ - public MFCC() { - this(13, 23, 133.0f); - } - - /** - * Constructor for MFCC. - * - * @param cepstra The number of cepstra to obtain for the MFCC feature. - * @param melfilters The number of triangular mel-filters (size of the mel-filter bank). - * @param minFrequency Minimum frequency to consider for MFCC feature. - */ - public MFCC(int cepstra, int melfilters, float minFrequency) { - this.melfilters = melfilters; - this.minFrequency = minFrequency; - this.cepstra = new float[cepstra]; + /** + * Minimum frequency to consider. Defaults to 100Hz as per [1]. + */ + private final float minFrequency; + + /** + * Number of Mel filters to use. Defaults to 23 as per [1]. + */ + private final int melfilters; + + /** + * Array holding the cepstrum coefficients (cepstra). Defaults to 13 as per [1]. + */ + private float[] cepstra; + + /** + * Default constructor for MFCC class with 13 cepstra, a Mel-Filter bank of size 13 and a minimum frequency of 133.0f Hz. + */ + public MFCC() { + this(13, 23, 133.0f); + } + + /** + * Constructor for MFCC. + * + * @param cepstra The number of cepstra to obtain for the MFCC feature. + * @param melfilters The number of triangular mel-filters (size of the mel-filter bank). + * @param minFrequency Minimum frequency to consider for MFCC feature. + */ + public MFCC(int cepstra, int melfilters, float minFrequency) { + this.melfilters = melfilters; + this.minFrequency = minFrequency; + this.cepstra = new float[cepstra]; + } + + /** + * Returns a list of MFCC features for the provided STFT using the default settings. + * + * @param stft STFT to derive the MFCC features from. + */ + public static List calculate(STFT stft) { + List spectra = stft.getMagnitudeSpectrum(); + List list = new ArrayList<>(spectra.size()); + for (Spectrum spectrum : spectra) { + MFCC mfcc = new MFCC(); + mfcc.calculate(spectrum, stft.getSamplingrate(), stft.getNumberOfBins()); + list.add(mfcc); } - - /** - * Returns a list of MFCC features for the provided STFT using the default settings. - * - * @param stft STFT to derive the MFCC features from. - */ - public static List calculate(STFT stft) { - List spectra = stft.getMagnitudeSpectrum(); - List list = new ArrayList<>(spectra.size()); - for (Spectrum spectrum : spectra) { - MFCC mfcc = new MFCC(); - mfcc.calculate(spectrum, stft.getSamplingrate(), stft.getNumberOfBins()); - list.add(mfcc); - } - return list; + return list; + } + + /** + * Returns a list of MFCC features for the provided STFT using the provided settings. + * + * @param stft STFT to derive the MFCC features from. + * @param cepstra The number of cepstra to obtain for the MFCC feature. + * @param melfilters The number of triangular mel-filters (size of the mel-filter bank). + * @param minFrequency Minimum frequency to consider for MFCC feature. + */ + public static List calculate(STFT stft, int cepstra, int melfilters, float minFrequency) { + List spectra = stft.getMagnitudeSpectrum(); + List list = new ArrayList<>(spectra.size()); + for (Spectrum spectrum : spectra) { + MFCC mfcc = new MFCC(cepstra, melfilters, minFrequency); + mfcc.calculate(spectrum, stft.getSamplingrate(), stft.getNumberOfBins()); + list.add(mfcc); } - - /** - * Returns a list of MFCC features for the provided STFT using the provided settings. - * - * @param stft STFT to derive the MFCC features from. - * @param cepstra The number of cepstra to obtain for the MFCC feature. - * @param melfilters The number of triangular mel-filters (size of the mel-filter bank). - * @param minFrequency Minimum frequency to consider for MFCC feature. - */ - public static List calculate(STFT stft, int cepstra, int melfilters, float minFrequency) { - List spectra = stft.getMagnitudeSpectrum(); - List list = new ArrayList<>(spectra.size()); - for (Spectrum spectrum : spectra) { - MFCC mfcc = new MFCC(cepstra, melfilters, minFrequency); - mfcc.calculate(spectrum, stft.getSamplingrate(), stft.getNumberOfBins()); - list.add(mfcc); - } - return list; + return list; + } + + /** + * Finds and returns the center frequency of the i-th mel filter out of a specified amount of filters. + * + * @param i The index of the filter for which a frequency should be obtained (zero based). + * @param total The total amount of mel filters. + * @param samplingrate The rate at which the original data has been sampled. + * @param minFrequency The minimum frequency to consider. + * @return Center frequency (normal) of the i-th mel filter. + */ + public static double centerFrequency(int i, int total, float samplingrate, float minFrequency) { + double minMel = frequencyToMel(minFrequency); + double maxMel = frequencyToMel(samplingrate / 2); + return melToFrequency(minMel + ((maxMel - minMel) / (total + 1)) * i); + } + + /** + * Transforms a frequency in Hz into a corresponding frequency on the mel scale and returns the value of that mel frequency. + * + * @param frequency Frequency to transform. + * @return Frequency on the mel scale corresponding to the normal frequency. + */ + public static double frequencyToMel(double frequency) { + return 2595 * Math.log10(1 + frequency / 700.0); + } + + /** + * Transforms a mel frequency into a frequency in Hz and returns the value of that frequency. + * + * @param mel Mel frequency to transform. + * @return Value corresponding to the normal frequency. + */ + public static double melToFrequency(double mel) { + return 700.0 * Math.pow(10, mel / 2595.0) - 700.0; + } + + public void calculate(Spectrum spectrum, float samplingrate, int windowsize) { + /* Check the type of the provided spectrum* */ + if (spectrum.getType() != Spectrum.Type.MAGNITUDE) { + throw new IllegalArgumentException("A magnitude spectrum is required in order to calculate the Mel Frequency Cepstrum Coefficients."); } - /** - * Finds and returns the center frequency of the i-th mel filter out of a specified amount - * of filters. - * - * @param i The index of the filter for which a frequency should be obtained (zero based). - * @param total The total amount of mel filters. - * @param samplingrate The rate at which the original data has been sampled. - * @param minFrequency The minimum frequency to consider. - * - * @return Center frequency (normal) of the i-th mel filter. - */ - public static double centerFrequency(int i, int total, float samplingrate, float minFrequency) { - double minMel = frequencyToMel(minFrequency); - double maxMel = frequencyToMel(samplingrate/2); - return melToFrequency(minMel + ((maxMel - minMel) / (total + 1)) * i); - } + /* */ + int[] bin_indices = this.melFrequencyBins(samplingrate, windowsize); - /** - * Transforms a frequency in Hz into a corresponding frequency on the mel scale and - * returns the value of that mel frequency. - * - * @param frequency Frequency to transform. - * @return Frequency on the mel scale corresponding to the normal frequency. - */ - public static double frequencyToMel(double frequency) { - return 2595 * Math.log10(1 + frequency/700.0); - } + /* Calculates and returns the Mel Filter Bank values of the power-spectrum. */ + double[] mel_filter_banks = this.melFilter(spectrum.array(), bin_indices); - /** - * Transforms a mel frequency into a frequency in Hz and returns the value of that - * frequency. - * - * @param mel Mel frequency to transform. - * @return Value corresponding to the normal frequency. - */ - public static double melToFrequency(double mel) { - return 700.0 * Math.pow(10, mel/2595.0) - 700.0; + /* Calculate and assign cepstra coefficients. */ + for (int i = 0; i < this.cepstra.length; i++) { + for (int j = 1; j <= this.melfilters; j++) { + this.cepstra[i] += mel_filter_banks[j - 1] * Math.cos(Math.PI * i / this.melfilters * (j - 0.5)); + } } - - public void calculate(Spectrum spectrum, float samplingrate, int windowsize) { - /* Check the type of the provided spectrum* */ - if (spectrum.getType() != Spectrum.Type.MAGNITUDE) { - throw new IllegalArgumentException("A magnitude spectrum is required in order to calculate the Mel Frequency Cepstrum Coefficients."); - } - - /* */ - int[] bin_indices = this.melFrequencyBins(samplingrate, windowsize); - - /* Calculates and returns the Mel Filter Bank values of the power-spectrum. */ - double[] mel_filter_banks = this.melFilter(spectrum.array(), bin_indices); - - /* Calculate and assign cepstra coefficients. */ - for (int i = 0; i < this.cepstra.length; i++){ - for (int j = 1; j <= this.melfilters; j++){ - this.cepstra[i] += mel_filter_banks[j - 1] * Math.cos(Math.PI * i / this.melfilters * (j - 0.5)); - } - } + } + + /** + * Calculates and returns the indices of the frequency-bins relevant for mel-filtering (relative to the full list of frequency bins). + * + * @param samplingrate The rate at which the original data has been sampled. + * @param windowsize Windowsize used for FFT. + * @return Array of bin-indices. + */ + public int[] melFrequencyBins(float samplingrate, int windowsize) { + int[] bins = new int[this.melfilters + 2]; + bins[0] = Math.round(this.minFrequency / samplingrate * windowsize); + bins[bins.length - 1] = (windowsize / 2); + for (int i = 1; i <= this.melfilters; i++) { + double f = centerFrequency(i, this.melfilters, samplingrate, this.minFrequency); + bins[i] = (int) Math.round(f / samplingrate * windowsize); } - - /** - * Calculates and returns the indices of the frequency-bins relevant for mel-filtering (relative - * to the full list of frequency bins). - * - * @param samplingrate The rate at which the original data has been sampled. - * @param windowsize Windowsize used for FFT. - * @return Array of bin-indices. - */ - public int[] melFrequencyBins(float samplingrate, int windowsize) { - int[] bins = new int[this.melfilters + 2]; - bins[0] = Math.round(this.minFrequency / samplingrate * windowsize); - bins[bins.length -1] = (windowsize/2); - for (int i = 1; i <= this.melfilters; i++){ - double f = centerFrequency(i, this.melfilters, samplingrate, this.minFrequency); - bins[i] = (int)Math.round(f / samplingrate * windowsize); - } - return bins; + return bins; + } + + /** + * Applies the triangular Mel-filters to the magnutude-spectrum + * + * @param magnitudes Linear magnitude spectrum to apply the mel-filters to. + * @param indices Indices to use as center-frequencies. + * @return Mel-scaled magnitude spectrum. + */ + private double[] melFilter(double magnitudes[], int indices[]) { + double temp[] = new double[this.melfilters + 2]; + + for (int k = 1; k <= this.melfilters; k++) { + double num1 = 0, num2 = 0; + + for (int i = indices[k - 1]; i <= indices[k]; i++) { + num1 += ((i - indices[k - 1] + 1) / (indices[k] - indices[k - 1] + 1)) * magnitudes[i]; + } + + for (int i = indices[k] + 1; i <= indices[k + 1]; i++) { + num2 += (1 - ((i - indices[k]) / (indices[k + 1] - indices[k] + 1))) * magnitudes[i]; + } + + temp[k] = num1 + num2; } - /** - * Applies the triangular Mel-filters to the magnutude-spectrum - * - * @param magnitudes Linear magnitude spectrum to apply the mel-filters to. - * @param indices Indices to use as center-frequencies. - * @return Mel-scaled magnitude spectrum. - */ - private double[] melFilter(double magnitudes[], int indices[]){ - double temp[] = new double[this.melfilters + 2]; - - for (int k = 1; k <= this.melfilters; k++){ - double num1 = 0, num2 = 0; - - for (int i = indices[k - 1]; i <= indices[k]; i++){ - num1 += ((i - indices[k - 1] + 1) / (indices[k] - indices[k-1] + 1)) * magnitudes[i]; - } - - for (int i = indices[k] + 1; i <= indices[k + 1]; i++){ - num2 += (1 - ((i - indices[k]) / (indices[k + 1] - indices[k] + 1))) * magnitudes[i]; - } - - temp[k] = num1 + num2; - } - - final double floor = -50.0; - double fbank[] = new double[this.melfilters]; - for (int i = 0; i < this.melfilters; i++){ - fbank[i] = Math.max(Math.log(temp[i + 1]), floor); - } - - return fbank; + final double floor = -50.0; + double fbank[] = new double[this.melfilters]; + for (int i = 0; i < this.melfilters; i++) { + fbank[i] = Math.max(Math.log(temp[i + 1]), floor); } - /** - * Getter for the cepstra. - * - * @return List of cepstra for the current MFCC. - */ - public float[] getCepstra() { - return cepstra; - } + return fbank; + } + + /** + * Getter for the cepstra. + * + * @return List of cepstra for the current MFCC. + */ + public float[] getCepstra() { + return cepstra; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Melody.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Melody.java index 90ed1ca88..079317c05 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Melody.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Melody.java @@ -1,118 +1,119 @@ package org.vitrivr.cineast.core.util.audio.pitch; -import org.vitrivr.cineast.core.util.dsp.FrequencyUtils; - import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; +import org.vitrivr.cineast.core.util.dsp.FrequencyUtils; /** * This class represents a melody as as sequence of pitches. - * */ public class Melody implements Iterable { - /** List holding the pitches that comprise this melody. */ - private final List pitches = new LinkedList<>(); - /** - * Appends a pitch to the melody stream. - * - * @param pitch Pitch to append. - */ - public void append(Pitch pitch) { - this.pitches.add(pitch); - } + /** + * List holding the pitches that comprise this melody. + */ + private final List pitches = new LinkedList<>(); - /** - * Prepends a pitch to the melody stream. - * - * @param pitch Pitch to prepend. - */ - public void prepend(Pitch pitch) { - this.pitches.add(0, pitch); - } + /** + * Appends a pitch to the melody stream. + * + * @param pitch Pitch to append. + */ + public void append(Pitch pitch) { + this.pitches.add(pitch); + } - /** - * Accessor for a pitch at a specific index. - * - * @param index Index of the pitch. - * @return Pitch - */ - public Pitch getPitch(int index) { - return this.pitches.get(index); - } + /** + * Prepends a pitch to the melody stream. + * + * @param pitch Pitch to prepend. + */ + public void prepend(Pitch pitch) { + this.pitches.add(0, pitch); + } - /** - * Returns the first pitch in the melody. - * - * @return First pitch in the melody. - */ - public final Pitch first() { - if (this.pitches.size() > 0) { - return this.pitches.get(0); - } else { - return null; - } - } + /** + * Accessor for a pitch at a specific index. + * + * @param index Index of the pitch. + * @return Pitch + */ + public Pitch getPitch(int index) { + return this.pitches.get(index); + } - /** - * Returns the last pitch in the melody. - * - * @return Last pitch in the melody. - */ - public final Pitch last() { - if (this.pitches.size() > 0) { - return this.pitches.get(this.pitches.size()-1); - } else { - return null; - } + /** + * Returns the first pitch in the melody. + * + * @return First pitch in the melody. + */ + public final Pitch first() { + if (this.pitches.size() > 0) { + return this.pitches.get(0); + } else { + return null; } + } - /** - * Returns a list of MIDI indices corresponding to the pitches in the melody. - * - * @return List of midi indices. - */ - public List getMidiIndices() { - return this.pitches.stream().map(Pitch::getIndex).collect(Collectors.toList()); + /** + * Returns the last pitch in the melody. + * + * @return Last pitch in the melody. + */ + public final Pitch last() { + if (this.pitches.size() > 0) { + return this.pitches.get(this.pitches.size() - 1); + } else { + return null; } + } - /** - * Returns a list of frequencies for every pitch. - * - * @return List of pitch-frequencies. - */ - public List getFrequencies() { - return this.pitches.stream().map(Pitch::getFrequency).collect(Collectors.toList()); - } + /** + * Returns a list of MIDI indices corresponding to the pitches in the melody. + * + * @return List of midi indices. + */ + public List getMidiIndices() { + return this.pitches.stream().map(Pitch::getIndex).collect(Collectors.toList()); + } - /** - * Returns the distance in cents of every pitch to some arbitrary minimum. - * - * @param min The minimum frequency to calculate the distance from in Hz. - * @return List of pitch-distances in cents. - */ - public List getCentDistances(float min) { - return this.pitches.stream().map(p -> FrequencyUtils.cents(p.getFrequency(), min)).collect(Collectors.toList()); - } + /** + * Returns a list of frequencies for every pitch. + * + * @return List of pitch-frequencies. + */ + public List getFrequencies() { + return this.pitches.stream().map(Pitch::getFrequency).collect(Collectors.toList()); + } - /** - * Returns the size of the melody in pitches. - * - * @return Size of the melody. - */ - public final int size() { - return this.pitches.size(); - } + /** + * Returns the distance in cents of every pitch to some arbitrary minimum. + * + * @param min The minimum frequency to calculate the distance from in Hz. + * @return List of pitch-distances in cents. + */ + public List getCentDistances(float min) { + return this.pitches.stream().map(p -> FrequencyUtils.cents(p.getFrequency(), min)).collect(Collectors.toList()); + } - /** - * Returns an iterator over elements of type {@code T}. - * - * @return an Iterator. - */ - @Override - public Iterator iterator() { - return this.pitches.iterator(); - } + /** + * Returns the size of the melody in pitches. + * + * @return Size of the melody. + */ + public final int size() { + return this.pitches.size(); + } + + /** + * Returns an iterator over elements of type {@code T}. + * + * @return an Iterator. + */ + @Override + public Iterator iterator() { + return this.pitches.iterator(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java index aa10f6237..de3875019 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/Pitch.java @@ -4,127 +4,132 @@ import org.vitrivr.cineast.core.util.dsp.midi.MidiUtil; /** - * This class represents a single pitch as returned for instance by pitch estimation classes or used by - * pitch tracking classes. A pitch can be either created based on a frequency in Hz or a MIDI index. - * + * This class represents a single pitch as returned for instance by pitch estimation classes or used by pitch tracking classes. A pitch can be either created based on a frequency in Hz or a MIDI index. */ public class Pitch { - /** MIDI index of the pitch. */ - private final int index; - - /** Frequency of the pitch in Hz. */ - private final float frequency; - - /** Salience of the pitch. */ - private double salience; - - /** Duration of the pitch in ms. Mainly used for playback. */ - private int duration = 500; - - /** - * Constructor of Pitch from a MIDI index. - * - * @param index MIDI index of the pitch. - */ - public Pitch(int index) { - if (index < 0) { - throw new IllegalArgumentException(""); - } - this.index = index; - this.frequency = MidiUtil.midiToFrequency(this.index); - } - - /** - * Constructor of Pitch from a frequency. - * - * @param frequency Frequency of the pitch in Hz. - */ - public Pitch(float frequency) { - this.frequency = frequency; - this.index = MidiUtil.frequencyToMidi(this.frequency); - } - - /** - * Getter for MIDI index. - * - * @return MIDI index of the pitch. - */ - public int getIndex() { - return index; - } - - /** - * Getter for frequency. - * - * @return Frequency of the pitch. - */ - public float getFrequency() { - return frequency; - } - - /** - * Getter for pitch salience. - */ - public double getSalience() { - return salience; - } - - /** - * Setter for pitch salience. - * - * @param salience New value for pitch salience. - */ - public void setSalience(double salience) { - this.salience = salience; - } - - /** - * Getter for duration. - * - * @return Pitch salience. - */ - public int getDuration() { - return duration; - } - - /** - * Setter for duration - * - * @param duration New value for duration. - */ - public void setDuration(int duration) { - this.duration = duration; - } - - /** - * Calculates the distance between the current pitch and the provided in Hz. - * - * @param pitch Pitch to measure the distance to. - * @return Distance in cents. - */ - public float distanceHertz(Pitch pitch) { - return Math.abs(this.frequency - pitch.frequency); - } - - /** - * Calculates the distance between the current pitch and the provided - * pitch on a cent scale. - * - * @param pitch Pitch to measure the distance to. - * @return Distance in cents. - */ - public double distanceCents(Pitch pitch) { - return FrequencyUtils.cents(pitch.frequency, this.frequency); - } - /** - * Calculates the distance between the current pitch and the provided - * pitch on a cent scale. - * - * @param frequency Frequency to measure the distance to. - * @return Distance in cents. - */ - public double distanceCents(float frequency) { - return FrequencyUtils.cents(frequency, this.frequency); + /** + * MIDI index of the pitch. + */ + private final int index; + + /** + * Frequency of the pitch in Hz. + */ + private final float frequency; + + /** + * Salience of the pitch. + */ + private double salience; + + /** + * Duration of the pitch in ms. Mainly used for playback. + */ + private int duration = 500; + + /** + * Constructor of Pitch from a MIDI index. + * + * @param index MIDI index of the pitch. + */ + public Pitch(int index) { + if (index < 0) { + throw new IllegalArgumentException(""); } + this.index = index; + this.frequency = MidiUtil.midiToFrequency(this.index); + } + + /** + * Constructor of Pitch from a frequency. + * + * @param frequency Frequency of the pitch in Hz. + */ + public Pitch(float frequency) { + this.frequency = frequency; + this.index = MidiUtil.frequencyToMidi(this.frequency); + } + + /** + * Getter for MIDI index. + * + * @return MIDI index of the pitch. + */ + public int getIndex() { + return index; + } + + /** + * Getter for frequency. + * + * @return Frequency of the pitch. + */ + public float getFrequency() { + return frequency; + } + + /** + * Getter for pitch salience. + */ + public double getSalience() { + return salience; + } + + /** + * Setter for pitch salience. + * + * @param salience New value for pitch salience. + */ + public void setSalience(double salience) { + this.salience = salience; + } + + /** + * Getter for duration. + * + * @return Pitch salience. + */ + public int getDuration() { + return duration; + } + + /** + * Setter for duration + * + * @param duration New value for duration. + */ + public void setDuration(int duration) { + this.duration = duration; + } + + /** + * Calculates the distance between the current pitch and the provided in Hz. + * + * @param pitch Pitch to measure the distance to. + * @return Distance in cents. + */ + public float distanceHertz(Pitch pitch) { + return Math.abs(this.frequency - pitch.frequency); + } + + /** + * Calculates the distance between the current pitch and the provided pitch on a cent scale. + * + * @param pitch Pitch to measure the distance to. + * @return Distance in cents. + */ + public double distanceCents(Pitch pitch) { + return FrequencyUtils.cents(pitch.frequency, this.frequency); + } + + /** + * Calculates the distance between the current pitch and the provided pitch on a cent scale. + * + * @param frequency Frequency to measure the distance to. + * @return Distance in cents. + */ + public double distanceCents(float frequency) { + return FrequencyUtils.cents(frequency, this.frequency); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java index 3cf3153b6..5865e7b5a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/estimation/KLF0PitchEstimator.java @@ -1,218 +1,226 @@ package org.vitrivr.cineast.core.util.audio.pitch.estimation; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import org.vitrivr.cineast.core.util.audio.pitch.Pitch; import org.vitrivr.cineast.core.util.dsp.fft.FFT; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; import org.vitrivr.cineast.core.util.dsp.midi.MidiUtil; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - /** - * This class can be used to estimate the most salient pitch(es) from a provided FFT or STFT by applying the method - * described in [1]. - * - * The method sums the amplitudes of pitch-candidates and their harmonic partials in a frame to obtain a salience function. - * The pitch candidate with the highest salience is then selected and removed from the spectrum. This procedure is repeated - * until all pitches have been detected. - * - * [1] Klapuri, A. (2006). Multiple Fundamental Frequency Estimation by Summing Harmonic Amplitudes. - * Proceedings of the International Symposium/Conference on Music Information Retrieval (ISMIR), 216–221. - * + * This class can be used to estimate the most salient pitch(es) from a provided FFT or STFT by applying the method described in [1]. + *

    + * The method sums the amplitudes of pitch-candidates and their harmonic partials in a frame to obtain a salience function. The pitch candidate with the highest salience is then selected and removed from the spectrum. This procedure is repeated until all pitches have been detected. + *

    + * [1] Klapuri, A. (2006). Multiple Fundamental Frequency Estimation by Summing Harmonic Amplitudes. Proceedings of the International Symposium/Conference on Music Information Retrieval (ISMIR), 216–221. */ public class KLF0PitchEstimator { - /** MIDI index of the minimum pitch to consider in the analysis. */ - private static final int DEFAULT_MIN_PITCH = 28; - - /** MIDI index of the maximum pitch to consider in the analysis. */ - private static final int DEFAULT_MAX_PITCH = 96; - - /** α value as defined in [1]. Suitable only for an analysis window of 96ms. */ - private static final float ALPHA = 27.0f; - - /** β value as defined in [1]. Suitable only for an analysis window of 96ms. */ - private static final float BETA = 320.0f; - - /** Maximum pitch to consider (MIDI index of the pitch). */ - private final int max; - - /** Maximum pitch to consider (MIDI index of the pitch). */ - private final int min; - - /** Maximum pitch to consider (MIDI index of the pitch). */ - private final float alpha; - /** Maximum pitch to consider (MIDI index of the pitch). */ - private final float beta; - - /** - * Default constructor for KLF0PitchEstimator. Uses parameter described in [1]. - */ - public KLF0PitchEstimator() { - this(DEFAULT_MIN_PITCH, DEFAULT_MAX_PITCH, ALPHA, BETA); - } - - public KLF0PitchEstimator(int min, int max, float alpha, float beta) { - this.max = max; - this.min = min; - this.alpha = alpha; - this.beta = beta; + /** + * MIDI index of the minimum pitch to consider in the analysis. + */ + private static final int DEFAULT_MIN_PITCH = 28; + + /** + * MIDI index of the maximum pitch to consider in the analysis. + */ + private static final int DEFAULT_MAX_PITCH = 96; + + /** + * α value as defined in [1]. Suitable only for an analysis window of 96ms. + */ + private static final float ALPHA = 27.0f; + + /** + * β value as defined in [1]. Suitable only for an analysis window of 96ms. + */ + private static final float BETA = 320.0f; + + /** + * Maximum pitch to consider (MIDI index of the pitch). + */ + private final int max; + + /** + * Maximum pitch to consider (MIDI index of the pitch). + */ + private final int min; + + /** + * Maximum pitch to consider (MIDI index of the pitch). + */ + private final float alpha; + + /** + * Maximum pitch to consider (MIDI index of the pitch). + */ + private final float beta; + + /** + * Default constructor for KLF0PitchEstimator. Uses parameter described in [1]. + */ + public KLF0PitchEstimator() { + this(DEFAULT_MIN_PITCH, DEFAULT_MAX_PITCH, ALPHA, BETA); + } + + public KLF0PitchEstimator(int min, int max, float alpha, float beta) { + this.max = max; + this.min = min; + this.alpha = alpha; + this.beta = beta; + } + + /** + * Estimates the pitches in the provided STFT and returns a List of PitchCandidate lists (one list per FFT). + * + * @param stft STFT for which to estimate the pitches. + * @return List of PitchCandidate lists. + */ + public List> estimatePitch(STFT stft) { + List> results = new ArrayList<>(stft.getStft().size()); + for (FFT fft : stft.getStft()) { + if (fft.isZero()) { + continue; + } + results.add(this.estimatePitch(fft)); } - - /** - * Estimates the pitches in the provided STFT and returns a List of PitchCandidate lists (one - * list per FFT). - * - * @param stft STFT for which to estimate the pitches. - * @return List of PitchCandidate lists. - */ - public List> estimatePitch(STFT stft) { - List> results = new ArrayList<>(stft.getStft().size()); - for (FFT fft : stft.getStft()) { - if (fft.isZero()) { - continue; - } - results.add(this.estimatePitch(fft)); + return results; + } + + + /** + * Estimates the pitches from the provided FFT and returns them as a list of PitchCandidates. + * + * @param fft FFT to estimate the pitches from. + * @return List of pitch candidates. + */ + public List estimatePitch(FFT fft) { + /* Prepare required helper variables. */ + final float samplingrate = fft.getSamplingrate(); + final int windowsize = fft.getWindowsize(); + final Spectrum spectrum = fft.getPowerSpectrum(); + + /* Prepare empty array of booleans holding the estimates. */ + List candidates = new ArrayList<>(); + + float test = 0, lasttest = 0; + int loopcount = 1; + while (true) { + /* Detect new candidate. */ + Pitch candidate = this.detect(spectrum, samplingrate, windowsize); + + /* Test if that candidate already exists. */ + boolean exists = false; + for (Pitch c : candidates) { + if (c.getFrequency() == candidate.getFrequency()) { + c.setSalience(candidate.getSalience() + c.getSalience()); + exists = true; + break; } - return results; - } - - - /** - * Estimates the pitches from the provided FFT and returns them as a list - * of PitchCandidates. - * - * @param fft FFT to estimate the pitches from. - * @return List of pitch candidates. - */ - public List estimatePitch(FFT fft) { - /* Prepare required helper variables. */ - final float samplingrate = fft.getSamplingrate(); - final int windowsize = fft.getWindowsize(); - final Spectrum spectrum = fft.getPowerSpectrum(); - - /* Prepare empty array of booleans holding the estimates. */ - List candidates = new ArrayList<>(); - - float test = 0, lasttest = 0; - int loopcount = 1; - while (true) { - /* Detect new candidate. */ - Pitch candidate = this.detect(spectrum, samplingrate, windowsize); - - /* Test if that candidate already exists. */ - boolean exists = false; - for (Pitch c : candidates) { - if (c.getFrequency() == candidate.getFrequency()) { - c.setSalience(candidate.getSalience() + c.getSalience()); - exists = true; - break; - } - } - - if (!exists) { - candidates.add(candidate); - } - - /* Conduct test and break if it fails. */ - lasttest = test; - test = (float)((test + candidate.getSalience()) / Math.pow(loopcount, .7f)); - if (test <= lasttest) { - break; - } - loopcount++; - - /* Add candidate to list. */ - candidates.add(candidate); - - final float f0 = candidate.getFrequency(); - final float tau = samplingrate/f0; /* Fundamental period, i.e. f0=fs/τ. */ - final float dtau = 0.25f; /* Δτ/2, which is 0.25 according to [1]. */ - - /* Subtract the information of the found pitch from the current spectrum. */ - for (int m = 1; m * candidate.getFrequency() < samplingrate / 2; m++) { - int max = Math.round((m*windowsize)/(tau - dtau)); - int min = Math.round((m*windowsize)/(tau + dtau)); - int max_bin = min; - for (int offset = min; offset <= max && offset < windowsize/2; offset++) { - if (spectrum.getValue(offset) > spectrum.getValue(max_bin)) { - max_bin = offset; - } - } - spectrum.setValue(max_bin, spectrum.getValue(max_bin) - spectrum.getValue(max_bin) * this.g(f0, m)); - } + } + + if (!exists) { + candidates.add(candidate); + } + + /* Conduct test and break if it fails. */ + lasttest = test; + test = (float) ((test + candidate.getSalience()) / Math.pow(loopcount, .7f)); + if (test <= lasttest) { + break; + } + loopcount++; + + /* Add candidate to list. */ + candidates.add(candidate); + + final float f0 = candidate.getFrequency(); + final float tau = samplingrate / f0; /* Fundamental period, i.e. f0=fs/τ. */ + final float dtau = 0.25f; /* Δτ/2, which is 0.25 according to [1]. */ + + /* Subtract the information of the found pitch from the current spectrum. */ + for (int m = 1; m * candidate.getFrequency() < samplingrate / 2; m++) { + int max = Math.round((m * windowsize) / (tau - dtau)); + int min = Math.round((m * windowsize) / (tau + dtau)); + int max_bin = min; + for (int offset = min; offset <= max && offset < windowsize / 2; offset++) { + if (spectrum.getValue(offset) > spectrum.getValue(max_bin)) { + max_bin = offset; + } } - - /* Sort list of candidates by their salience in descending order. */ - candidates.sort(Comparator.comparingDouble(Pitch::getSalience)); - Collections.reverse(candidates); - - /* Return list of candidates. */ - return candidates; + spectrum.setValue(max_bin, spectrum.getValue(max_bin) - spectrum.getValue(max_bin) * this.g(f0, m)); + } } - /** - * Detects the most salient F0 candidate in the provided spectrum. - * - * @param spectrum Power spectrum to search for the candidate. - * @param samplingrate Samplingrate at which the original signal has been sampled. - * @param windowsize Windowsize used in the FFT. - */ - private Pitch detect(Spectrum spectrum, final float samplingrate, final int windowsize) { - Pitch candidate = null; - for (int n = this.min; n<= this.max; n++) { - final float f0 = MidiUtil.midiToFrequency(n); - final double salience = this.salience(f0, spectrum, samplingrate, windowsize); - if (candidate == null || candidate.getSalience() < salience) { - candidate = new Pitch(f0); - candidate.setSalience(salience); - } - } - return candidate; + /* Sort list of candidates by their salience in descending order. */ + candidates.sort(Comparator.comparingDouble(Pitch::getSalience)); + Collections.reverse(candidates); + + /* Return list of candidates. */ + return candidates; + } + + /** + * Detects the most salient F0 candidate in the provided spectrum. + * + * @param spectrum Power spectrum to search for the candidate. + * @param samplingrate Samplingrate at which the original signal has been sampled. + * @param windowsize Windowsize used in the FFT. + */ + private Pitch detect(Spectrum spectrum, final float samplingrate, final int windowsize) { + Pitch candidate = null; + for (int n = this.min; n <= this.max; n++) { + final float f0 = MidiUtil.midiToFrequency(n); + final double salience = this.salience(f0, spectrum, samplingrate, windowsize); + if (candidate == null || candidate.getSalience() < salience) { + candidate = new Pitch(f0); + candidate.setSalience(salience); + } } - - /** - * Calculates and returns the salience of a f0 in a spectrum according to [1]. - * - * @param f0 The f0 to calculate the salience for. - * @param spectrum The spectrum to check. - * @param samplingrate The rate at which the original audio has been sampled. - * @param windowsize The windowsize used during creation of the spectrum. - * @return Salience of f0 in the spectrum. - */ - private final double salience(float f0, Spectrum spectrum, final float samplingrate, final int windowsize) { - final float tau = samplingrate/f0; /* Fundamental period, i.e. f0=fs/τ. */ - final float dtau = 0.25f; /* Δτ/2, which is 0.25 according to [1]. */ - float salience = 0; /* Salience of the candidate pitch. */ - - for (int m = 1; m * f0 < samplingrate/2; m++) { - int max = Math.round((m*windowsize)/(tau - dtau)); - int min = Math.round((m*windowsize)/(tau + dtau)); - int max_bin = min; - for (int offset = min; offset <= max && offset < windowsize/2; offset++) { - if (spectrum.getValue(offset) > spectrum.getValue(max_bin)) { - max_bin = offset; - } - } - salience += spectrum.getValue(max_bin) * this.g(f0, m); + return candidate; + } + + /** + * Calculates and returns the salience of a f0 in a spectrum according to [1]. + * + * @param f0 The f0 to calculate the salience for. + * @param spectrum The spectrum to check. + * @param samplingrate The rate at which the original audio has been sampled. + * @param windowsize The windowsize used during creation of the spectrum. + * @return Salience of f0 in the spectrum. + */ + private final double salience(float f0, Spectrum spectrum, final float samplingrate, final int windowsize) { + final float tau = samplingrate / f0; /* Fundamental period, i.e. f0=fs/τ. */ + final float dtau = 0.25f; /* Δτ/2, which is 0.25 according to [1]. */ + float salience = 0; /* Salience of the candidate pitch. */ + + for (int m = 1; m * f0 < samplingrate / 2; m++) { + int max = Math.round((m * windowsize) / (tau - dtau)); + int min = Math.round((m * windowsize) / (tau + dtau)); + int max_bin = min; + for (int offset = min; offset <= max && offset < windowsize / 2; offset++) { + if (spectrum.getValue(offset) > spectrum.getValue(max_bin)) { + max_bin = offset; } - - return salience; + } + salience += spectrum.getValue(max_bin) * this.g(f0, m); } + return salience; + } - /** - * Returns the value of the weight-function for pitch-salience calculation - * according to [1]. - * - * @param f0 Fundamental frequency of the pitch candidate. - * @param m Index of the partial (i.e. m-th partial -> fm = m*f0) - * @return weight for the provided f0/partial constellation. - */ - private double g(float f0, int m) { - return (f0 + this.alpha)/(m*f0 + this.beta); - } + + /** + * Returns the value of the weight-function for pitch-salience calculation according to [1]. + * + * @param f0 Fundamental frequency of the pitch candidate. + * @param m Index of the partial (i.e. m-th partial -> fm = m*f0) + * @return weight for the provided f0/partial constellation. + */ + private double g(float f0, int m) { + return (f0 + this.alpha) / (m * f0 + this.beta); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java index 1df7d53e5..c03891dd9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchContour.java @@ -1,211 +1,217 @@ package org.vitrivr.cineast.core.util.audio.pitch.tracking; -import org.apache.commons.math3.stat.descriptive.SummaryStatistics; -import org.vitrivr.cineast.core.util.audio.pitch.Pitch; - import java.util.LinkedList; import java.util.List; +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.vitrivr.cineast.core.util.audio.pitch.Pitch; /** - * This is a helper class for pitch tracking. It represents a pitch contour, that is, a candidate for a melody fragment. The contour - * has a fixed length and each slot in the sequence represents a specific timeframe (e.g. belonging to a FFT bin in the underlying STFT). - * - * The intention behind this class is the simplification of comparison between different pitch contours, either on a frame-by-frame basis - * but also as an entity. In addition to the actual pitch information, the pitch contour class also provides access to pitch contour statistics - * related to salience and pitch frequency. + * This is a helper class for pitch tracking. It represents a pitch contour, that is, a candidate for a melody fragment. The contour has a fixed length and each slot in the sequence represents a specific timeframe (e.g. belonging to a FFT bin in the underlying STFT). + *

    + * The intention behind this class is the simplification of comparison between different pitch contours, either on a frame-by-frame basis but also as an entity. In addition to the actual pitch information, the pitch contour class also provides access to pitch contour statistics related to salience and pitch frequency. * * @see PitchTracker - * */ public class PitchContour { - /** The minimum frequency in Hz on the (artifical) cent-scale. */ - private static final float CENT_SCALE_MINIMUM = 55.0f; - - /** Entity that keeps track of salience related contour statistics. */ - private SummaryStatistics salienceStatistics = new SummaryStatistics(); - - /** Entity that keeps track of frequency related contour statistics. */ - private SummaryStatistics frequencyStatistics = new SummaryStatistics(); - - /** Sequence of pitches that form the PitchContour. */ - private final List contour = new LinkedList<>(); - - /** Indicates that the PitchContour statistics require recalculation. */ - private boolean dirty = true; - - /** The start frame-index of the pitch-contour. Marks beginning in time. */ - private int start; - - /** The end frame-index of the pitch-contour. Marks ending in time. */ - private int end; - - /** - * Constructor for PitchContour. - * - * @param start Start-index of the contour. - * @param pitch Pitch that belongs to the start-index. - */ - public PitchContour(int start, Pitch pitch) { - this.start = start; - this.end = start; - this.contour.add(pitch); - } - - /** - * Sets the pitch at the given index if the index is within the bounds - * of the PitchContour. - * - * @param p Pitch to append. - */ - public void append(Pitch p) { - this.contour.add(p); - this.end += 1; - this.dirty = true; - } - - /** - * Sets the pitch at the given index if the index is within the bounds - * of the PitchContour. - * - * @param p Pitch to append. - */ - public void prepend(Pitch p) { - this.contour.add(0, p); - this.start -= 1; - this.dirty = true; - } - - /** - * Returns the pitch at the given index or null, if the index is out of bounds. - * Note that even if the index is within bounds, the Pitch can still be null. - * - * @param i Index for which to return a pitch. - */ - public Pitch getPitch(int i) { - if (i >= this.start && i <= this.end) { - return this.contour.get(i-this.start); - } else { - return null; - } - } - - /** - * Getter for start. - * - * @return Start frame-index. - */ - public final int getStart() { - return start; - } - - /** - * Getter for end. - * - * @return End frame-index. - */ - public final int getEnd() { - return end; - } - - - /** - * Size of the pitch-contour. This number also includes - * empty slots. - * - * @return Size of the contour. - */ - public final int size() { - return this.contour.size(); - } - /** - * Returns the mean of all pitches in the melody. - * - * @return Pitch mean - */ - public final double pitchMean() { - if (this.dirty) { - this.calculate(); - } - return this.frequencyStatistics.getMean(); + /** + * The minimum frequency in Hz on the (artifical) cent-scale. + */ + private static final float CENT_SCALE_MINIMUM = 55.0f; + + /** + * Entity that keeps track of salience related contour statistics. + */ + private SummaryStatistics salienceStatistics = new SummaryStatistics(); + + /** + * Entity that keeps track of frequency related contour statistics. + */ + private SummaryStatistics frequencyStatistics = new SummaryStatistics(); + + /** + * Sequence of pitches that form the PitchContour. + */ + private final List contour = new LinkedList<>(); + + /** + * Indicates that the PitchContour statistics require recalculation. + */ + private boolean dirty = true; + + /** + * The start frame-index of the pitch-contour. Marks beginning in time. + */ + private int start; + + /** + * The end frame-index of the pitch-contour. Marks ending in time. + */ + private int end; + + /** + * Constructor for PitchContour. + * + * @param start Start-index of the contour. + * @param pitch Pitch that belongs to the start-index. + */ + public PitchContour(int start, Pitch pitch) { + this.start = start; + this.end = start; + this.contour.add(pitch); + } + + /** + * Sets the pitch at the given index if the index is within the bounds of the PitchContour. + * + * @param p Pitch to append. + */ + public void append(Pitch p) { + this.contour.add(p); + this.end += 1; + this.dirty = true; + } + + /** + * Sets the pitch at the given index if the index is within the bounds of the PitchContour. + * + * @param p Pitch to append. + */ + public void prepend(Pitch p) { + this.contour.add(0, p); + this.start -= 1; + this.dirty = true; + } + + /** + * Returns the pitch at the given index or null, if the index is out of bounds. Note that even if the index is within bounds, the Pitch can still be null. + * + * @param i Index for which to return a pitch. + */ + public Pitch getPitch(int i) { + if (i >= this.start && i <= this.end) { + return this.contour.get(i - this.start); + } else { + return null; } - - /** - * Returns the standard-deviation of all pitches in the melody. - * - * @return Pitch standard deviation - */ - public final double pitchDeviation() { - if (this.dirty) { - this.calculate(); - } - return this.frequencyStatistics.getStandardDeviation(); + } + + /** + * Getter for start. + * + * @return Start frame-index. + */ + public final int getStart() { + return start; + } + + /** + * Getter for end. + * + * @return End frame-index. + */ + public final int getEnd() { + return end; + } + + + /** + * Size of the pitch-contour. This number also includes empty slots. + * + * @return Size of the contour. + */ + public final int size() { + return this.contour.size(); + } + + /** + * Returns the mean of all pitches in the melody. + * + * @return Pitch mean + */ + public final double pitchMean() { + if (this.dirty) { + this.calculate(); } - - /** - * Returns the mean-salience of all pitches in the contour. - * - * @return Salience mean - */ - public final double salienceMean() { - if (this.dirty) { - this.calculate(); - } - return this.salienceStatistics.getMean(); + return this.frequencyStatistics.getMean(); + } + + /** + * Returns the standard-deviation of all pitches in the melody. + * + * @return Pitch standard deviation + */ + public final double pitchDeviation() { + if (this.dirty) { + this.calculate(); } - - /** - * Returns the salience standard deviation of all pitches in the contour. - * - * @return Salience standard deviation. - */ - public final double salienceDeviation() { - if (this.dirty) { - this.calculate(); - } - return this.salienceStatistics.getStandardDeviation(); + return this.frequencyStatistics.getStandardDeviation(); + } + + /** + * Returns the mean-salience of all pitches in the contour. + * + * @return Salience mean + */ + public final double salienceMean() { + if (this.dirty) { + this.calculate(); } - - /** - * Returns the sum of all salience values in the pitch contour. - */ - public final double salienceSum() { - if (this.dirty) { - this.calculate(); - } - return this.salienceStatistics.getSum(); + return this.salienceStatistics.getMean(); + } + + /** + * Returns the salience standard deviation of all pitches in the contour. + * + * @return Salience standard deviation. + */ + public final double salienceDeviation() { + if (this.dirty) { + this.calculate(); } - - /** - * Calculates the overlap between the given pitch-contours. - * - * @return Size of the overlap between two pitch-contours. - */ - public final int overlap(PitchContour contour) { - return Math.max(0, Math.min(this.end,contour.end) - Math.max(this.start, contour.start)); - } - - /** - * Determines if two PitchContours overlap and returns true of false. - * - * @return true, if two PitchContours overlap and falseotherwise. - */ - public final boolean overlaps(PitchContour contour) { - return this.overlap(contour) > 0; + return this.salienceStatistics.getStandardDeviation(); + } + + /** + * Returns the sum of all salience values in the pitch contour. + */ + public final double salienceSum() { + if (this.dirty) { + this.calculate(); } - - /** - * Re-calculates the PitchContour statistics. - */ - private void calculate() { - this.salienceStatistics.clear(); - this.frequencyStatistics.clear(); - for (Pitch pitch : this.contour) { - if (pitch != null) { - this.salienceStatistics.addValue(pitch.getSalience()); - this.frequencyStatistics.addValue(pitch.distanceCents(CENT_SCALE_MINIMUM)); - } - } - this.dirty = false; + return this.salienceStatistics.getSum(); + } + + /** + * Calculates the overlap between the given pitch-contours. + * + * @return Size of the overlap between two pitch-contours. + */ + public final int overlap(PitchContour contour) { + return Math.max(0, Math.min(this.end, contour.end) - Math.max(this.start, contour.start)); + } + + /** + * Determines if two PitchContours overlap and returns true of false. + * + * @return true, if two PitchContours overlap and falseotherwise. + */ + public final boolean overlaps(PitchContour contour) { + return this.overlap(contour) > 0; + } + + /** + * Re-calculates the PitchContour statistics. + */ + private void calculate() { + this.salienceStatistics.clear(); + this.frequencyStatistics.clear(); + for (Pitch pitch : this.contour) { + if (pitch != null) { + this.salienceStatistics.addValue(pitch.getSalience()); + this.frequencyStatistics.addValue(pitch.distanceCents(CENT_SCALE_MINIMUM)); + } } + this.dirty = false; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java index 78fc3b698..f3b6fa404 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/audio/pitch/tracking/PitchTracker.java @@ -1,578 +1,592 @@ package org.vitrivr.cineast.core.util.audio.pitch.tracking; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.vitrivr.cineast.core.util.audio.pitch.Melody; import org.vitrivr.cineast.core.util.audio.pitch.Pitch; import org.vitrivr.cineast.core.util.dsp.FrequencyUtils; -import java.util.*; - /** - * This class implements the pitch-tracking algorithm described in [1]. It can be used to extract a melody from a - * list of pitch-candidates. The list of pitch-candidates must be extracted using a pitch-extraction method. - * - * [1] Salamon, J., & Gomez, E. (2012). Melody extraction from polyphonic music signals using pitch contour characteristics. - * IEEE Transactions on Audio, Speech and Language Processing, 20(6), 1759–1770. http://doi.org/10.1109/TASL.2012.2188515 - * + * This class implements the pitch-tracking algorithm described in [1]. It can be used to extract a melody from a list of pitch-candidates. The list of pitch-candidates must be extracted using a pitch-extraction method. + *

    + * [1] Salamon, J., & Gomez, E. (2012). Melody extraction from polyphonic music signals using pitch contour characteristics. IEEE Transactions on Audio, Speech and Language Processing, 20(6), 1759–1770. http://doi.org/10.1109/TASL.2012.2188515 */ public class PitchTracker { - /** Pitch-candidates that have been ruled out. */ - private Pitch[][] s0; - /** Active pitch-candidates. */ - private Pitch[][] s1; + /** + * Pitch-candidates that have been ruled out. + */ + private Pitch[][] s0; + + /** + * Active pitch-candidates. + */ + private Pitch[][] s1; + + /** + * Number of ms that passes between two adjacent bins. + */ + private float t_stepsize; + + /** + * Threshold in first (per-frame) filter step. + */ + private final float t1; + + /** + * Threshold in second (global) filter step. + */ + private final float t2; + + /** + * Voicing threshold according to [1]. + */ + private final float v_threshold; + + /** + * Maximum distance between two pitches in subsequent frames (in cents) according to [1]. + */ + private final float d_max; + + /** + * Maximum amount of misses in seconds before contour tracking is aborted. + */ + private final float m_max; + + /** + * SummaryStatistics for the PitchContours. + */ + private final SummaryStatistics contourStatistics = new SummaryStatistics(); + + /** + * List of PitchContours. This list is generated int the pitch-tracking step. + */ + private final List pitchContours = new ArrayList<>(); + + /** + * Default constructor for PitchTracker. Uses the settings described in [1]. + */ + public PitchTracker() { + this(0.9f, 0.9f, 0.2f, 80.0f, 0.1f); + } + + /** + * Constructor for PitchTracker class. + * + * @param t1 Threshold in first (per-frame) filter step. + * @param t2 Threshold in second (global) filter step. + * @param v_threshold Threshold for voicing detection. + * @param d_max Maximum distance between two pitches in subsequent frames during tracking. + * @param m_max Maximum amount of misses before contour tracking is aborted. + */ + public PitchTracker(float t1, float t2, float v_threshold, float d_max, float m_max) { + this.t1 = t1; + this.t2 = t2; + this.v_threshold = v_threshold; + this.d_max = d_max; + this.m_max = m_max; + } + + /** + * Initializes the PitchTracker with a set of new pitch candidates. Prepares everything for pitch-streaming and subsequent melody selection and discards previous results. + * + * @param candidates Nested list of pitch candidates. The first list contains one list per time-frame, each time-frame then contains one to n pitch-candidates. + * @param t_stepsize The time in milliseconds that passes between to adjacent bins. + */ + public void initialize(List> candidates, float t_stepsize) { + /* Initialize S1 and S0 with new pitch-candidates */ + this.s1 = new Pitch[candidates.size()][]; + this.s0 = new Pitch[candidates.size()][]; + this.t_stepsize = t_stepsize; + for (int i = 0; i < candidates.size(); i++) { + int size = candidates.get(i).size(); + this.s1[i] = new Pitch[size]; + this.s0[i] = new Pitch[size]; + for (int j = 0; j < size; j++) { + this.s1[i][j] = candidates.get(i).get(j); + } + } - /** Number of ms that passes between two adjacent bins. */ - private float t_stepsize; + /* Clear pre-calculates statistics and contours. */ + this.contourStatistics.clear(); + this.pitchContours.clear(); + } + + /** + * Executes the pitch-tracking / pitch-streaming step + */ + public void trackPitches() { + /* Apply the two filter stages described in [1], section II-B. */ + if (this.t1 > 0) { + this.applyPerFrameFilter(); + } + if (this.t2 > 0) { + this.applyGlobalFilter(); + } - /** Threshold in first (per-frame) filter step. */ - private final float t1; + int[] currentPointer = null; + Pitch currentPitch = null; + while (true) { + /* Find the pitch with the maximum salience in S1 and add it to the melody. */ + currentPointer = this.seekMostSalientInS1(); + if (currentPointer[0] == -1 || currentPointer[1] == -1) { + break; + } - /** Threshold in second (global) filter step. */ - private final float t2; + /* Select that pitch from S1 and create new PitchContour. */ + currentPitch = this.selectFromS1(currentPointer[0], currentPointer[1]); + final PitchContour contour = new PitchContour(currentPointer[0], currentPitch); - /** Voicing threshold according to [1]. */ - private final float v_threshold; + /* Track pitch contour. */ + this.track(contour, currentPointer[0]); - /** Maximum distance between two pitches in subsequent frames (in cents) according to [1]. */ - private final float d_max; + /* Add contour to list of contours */ + this.addContour(contour); + } - /** Maximum amount of misses in seconds before contour tracking is aborted. */ - private final float m_max; + /* + * Sort the list of pitch-contours so that the PitchContour with the highest salience-sum + * is the first entry. + */ + this.pitchContours.sort(Comparator.comparingDouble(PitchContour::salienceSum)); + Collections.reverse(this.pitchContours); + } + + public Melody extractMelody(int iterations) { + /* Return if no pitch-contours are available. */ + if (this.pitchContours.isEmpty()) { + return null; + } - /** SummaryStatistics for the PitchContours. */ - private final SummaryStatistics contourStatistics = new SummaryStatistics(); + /* Copies the pitch-contours from the tracking step. */ + List workingList = new ArrayList<>(this.pitchContours); - /** List of PitchContours. This list is generated int the pitch-tracking step.*/ - private final List pitchContours = new ArrayList<>(); + /* Filters the workingList and removes unvoiced pitch-contours. */ + this.voicingDetection(workingList); - /** - * Default constructor for PitchTracker. Uses the settings described in [1]. + /* + * Use the mean-contour to remove octave-duplicates and pitch-outliers. Perform multiple + * iterations in which the mean-contour is updated. */ - public PitchTracker() { - this(0.9f, 0.9f,0.2f, 80.0f, 0.1f); + double[] mean = this.meanContour(this.pitchContours); + List copy = null; + for (int i = 0; i < iterations; i++) { + copy = new ArrayList<>(workingList); + this.detectAndRemoveOctaveDuplicates(copy, mean); + mean = this.meanContour(copy); + this.detectAndRemovePitchOutliers(copy, mean); + mean = this.meanContour(copy); } - /** - * Constructor for PitchTracker class. - * - * @param t1 Threshold in first (per-frame) filter step. - * @param t2 Threshold in second (global) filter step. - * @param v_threshold Threshold for voicing detection. - * @param d_max Maximum distance between two pitches in subsequent frames during tracking. - * @param m_max Maximum amount of misses before contour tracking is aborted. + /* + * Sort the copied-list so that the PitchContour with the highest salience-sum is + * the first entry. */ - public PitchTracker(float t1, float t2, float v_threshold, float d_max, float m_max) { - this.t1 = t1; - this.t2 = t2; - this.v_threshold = v_threshold; - this.d_max = d_max; - this.m_max = m_max; - } + copy.sort(Comparator.comparingDouble(PitchContour::salienceSum)); + Collections.reverse(workingList); - /** - * Initializes the PitchTracker with a set of new pitch candidates. Prepares everything - * for pitch-streaming and subsequent melody selection and discards previous results. - * - * @param candidates Nested list of pitch candidates. The first list contains one list per time-frame, each time-frame then contains one to n pitch-candidates. - * @param t_stepsize The time in milliseconds that passes between to adjacent bins. + /* + * Construct melody from remaining pitch-contours. */ - public void initialize(List> candidates, float t_stepsize) { - /* Initialize S1 and S0 with new pitch-candidates */ - this.s1 = new Pitch[candidates.size()][]; - this.s0 = new Pitch[candidates.size()][]; - this.t_stepsize = t_stepsize; - for (int i=0; i 0.1f) { + melodyPitch.setDuration((int) (time * 1000)); + melodyPitch.setSalience(salience); + + melody.append(melodyPitch); + i = j - 1; + } + break; } - - /* Clear pre-calculates statistics and contours. */ - this.contourStatistics.clear(); - this.pitchContours.clear(); + } } - - /** - * Executes the pitch-tracking / pitch-streaming step - */ - public void trackPitches() { - /* Apply the two filter stages described in [1], section II-B. */ - if (this.t1 > 0) { - this.applyPerFrameFilter(); - } - if (this.t2 > 0) { - this.applyGlobalFilter(); - } - - int[] currentPointer = null; - Pitch currentPitch = null; - while (true) { - /* Find the pitch with the maximum salience in S1 and add it to the melody. */ - currentPointer = this.seekMostSalientInS1(); - if (currentPointer[0] == -1 || currentPointer[1] == -1) { - break; - } - - /* Select that pitch from S1 and create new PitchContour. */ - currentPitch = this.selectFromS1(currentPointer[0], currentPointer[1]); - final PitchContour contour = new PitchContour(currentPointer[0], currentPitch); - - /* Track pitch contour. */ - this.track(contour, currentPointer[0]); - - /* Add contour to list of contours */ - this.addContour(contour); - } - - /* - * Sort the list of pitch-contours so that the PitchContour with the highest salience-sum - * is the first entry. - */ - this.pitchContours.sort(Comparator.comparingDouble(PitchContour::salienceSum)); - Collections.reverse( this.pitchContours); + return melody; + } + + public double[] meanContour(List contours) { + + final int size = 40; + + /* Calculate pitch-mean. */ + double[] framesum = new double[this.s1.length]; + double[] weights = new double[this.s1.length]; + double[] pitchmean = new double[this.s1.length]; + for (PitchContour contour : contours) { + for (int i = contour.getStart(); i < contour.getEnd(); i++) { + framesum[i] += contour.getPitch(i).getFrequency() * contour.salienceSum(); + weights[i] += contour.salienceSum(); + } } - public Melody extractMelody(int iterations) { - /* Return if no pitch-contours are available. */ - if (this.pitchContours.isEmpty()) { - return null; - } - - /* Copies the pitch-contours from the tracking step. */ - List workingList = new ArrayList<>(this.pitchContours); - - /* Filters the workingList and removes unvoiced pitch-contours. */ - this.voicingDetection(workingList); - - /* - * Use the mean-contour to remove octave-duplicates and pitch-outliers. Perform multiple - * iterations in which the mean-contour is updated. - */ - double[] mean = this.meanContour(this.pitchContours); - List copy = null; - for (int i=0; i(workingList); - this.detectAndRemoveOctaveDuplicates(copy, mean); - mean = this.meanContour(copy); - this.detectAndRemovePitchOutliers(copy, mean); - mean = this.meanContour(copy); - } - - /* - * Sort the copied-list so that the PitchContour with the highest salience-sum is - * the first entry. - */ - copy.sort(Comparator.comparingDouble(PitchContour::salienceSum)); - Collections.reverse(workingList); - - /* - * Construct melody from remaining pitch-contours. - */ - Melody melody = new Melody(); - for (int i=0;i 0.1f) { - melodyPitch.setDuration((int)(time * 1000)); - melodyPitch.setSalience(salience); - - melody.append(melodyPitch); - i=j-1; - } - break; - } - } + for (int i = 0; i < framesum.length; i++) { + int start = Math.max(0, i - size / 2); + int end = Math.min(framesum.length, i + size / 2); + for (int k = start; k < end; k++) { + if (weights[k] > 0) { + pitchmean[i] += framesum[k] / weights[k]; } - return melody; + } + pitchmean[i] /= (end - start + 1); } - public double[] meanContour(List contours) { - - final int size = 40; - - /* Calculate pitch-mean. */ - double[] framesum = new double[this.s1.length]; - double[] weights = new double[this.s1.length]; - double[] pitchmean = new double[this.s1.length]; - for (PitchContour contour : contours) { - for (int i = contour.getStart(); i this.s1[max[0]][max[1]].getSalience()) { + max[0] = t; + max[1] = max_i; + } + } + return max; + } + + /** + * Seeks the pitch with the maximum salience in the specified frame in S1 and returns the index i that point to that maximum. If now maximum was found, -1 is returned. + * + * @param t Temporal index of the frame in S1. + * @return Index of the maximum in the specified frame. + */ + private int seekMostSalientInFrameS1(int t) { + Pitch[] pitches = this.s1[t]; + int max = -1; + for (int i = 0; i < pitches.length; i++) { + if (pitches[i] == null) { + continue; + } + if (max == -1 || pitches[i].getSalience() > pitches[max].getSalience()) { + max = i; + } + } + return max; + } + + /** + * Adds a pitch-contour to the list of contours and updates the contour-statistics. + * + * @param contour PitchContour to add. + */ + private void addContour(PitchContour contour) { + this.pitchContours.add(contour); + this.contourStatistics.addValue(contour.salienceMean()); + } + + /** + * Applies a per-frame filter on pitches in S1 and moves all peaks whose salience is bellow a certain threshold from S1 to S0. This filter is described in [1], section II-C. + */ + private void applyPerFrameFilter() { + for (int t = 0; t < this.s1.length; t++) { + int max_idx = this.seekMostSalientInFrameS1(t); + if (max_idx == -1) { + continue; + } + int size = this.s1[t].length; + for (int i = 0; i < size; i++) { + if (this.s1[t][i] == null) { + continue; } - - for (int i=0; i 0) { - pitchmean[i] += framesum[k]/weights[k]; - } - } - pitchmean[i] /= (end-start+1); + if (this.s1[t][i].getSalience() < this.t1 * this.s1[t][max_idx].getSalience()) { + this.moveToS0(t, i); } - - return pitchmean; + } } - - /** - * Selects the pitch specified by the two pitch-indices from S1 a and returns - * it, if it exists. Thereby, the pitch is removed from S1. - * - * @param t Temporal index of the pitch candidate. - * @param i Position of the pitch candidate - */ - private Pitch selectFromS1(int t, int i) { - Pitch pitch = this.s1[t][i]; - if (pitch != null) { - this.s1[t][i] = null; - return pitch; - } else { - return null; + } + + /** + * Applies a global filter on pitches in S1 and moves all pitches whose salience is bellow a certain threshold from S1 to S0. This filter is described in [1], section II-C. + */ + private void applyGlobalFilter() { + SummaryStatistics statistics = new SummaryStatistics(); + + /* Iteration #1: Gather data to obtain salience statistics. */ + for (int t = 0; t < this.s1.length; t++) { + for (int i = 0; i < this.s1[t].length; i++) { + if (this.s1[t][i] == null) { + continue; } + statistics.addValue(this.s1[t][i].getSalience()); + } } - /** - * Selects the pitch specified by the two pitch-indices from S0 a and returns - * it, if it exists. Thereby, the pitch is removed from S0. - * - * @param t Temporal index of the pitch candidate. - * @param i Position of the pitch candidate - */ - private Pitch selectFromS0(int t, int i) { - Pitch pitch = this.s0[t][i]; - if (pitch != null) { - this.s0[t][i] = null; - return pitch; + /* Iteration #2: Move pitches that are bellow the threshold. */ + final double threshold = statistics.getMean() - this.t2 * statistics.getStandardDeviation(); + for (int t = 0; t < this.s1.length; t++) { + for (int i = 0; i < this.s1[t].length; i++) { + if (this.s1[t][i] == null) { + continue; } - return null; + if (this.s1[t][i].getSalience() < threshold) { + this.moveToS0(t, i); + } + } } + } - /** - * Moves a pitch-candidate from S1 to S0. - * - * @param t Temporal index of the pitch candidate. - * @param i Position of the pitch candidate - * @return true if pitch was moved and false otherwise. - */ - private boolean moveToS0(int t, int i) { - if (this.s1[t][i] != null) { - this.s0[t][i] = this.s1[t][i]; - this.s1[t][i] = null; - return true; - } else { - return false; - } + private void track(final PitchContour contour, final int start) { + /* If start is the last entry, then no forward-tracking is required. */ + if (start == this.s1.length - 1) { + return; } + /* Initialize helper variables; number of pitches and last-pitch. */ + int misses = 0; + Pitch lastPitch = contour.getPitch(start); - /** - * Seeks the pitch with maximum salience in S1 and returns an int array that contains the - * indexes which point to that maximum. If no maximum was found, {-1, -1} is returned. - * - * @return Indexes {t,i} pointing to maximum is S1. - */ - private int[] seekMostSalientInS1() { - int[] max = {-1, -1}; - for (int t = 0; t this.s1[max[0]][max[1]].getSalience()) { - max[0] = t; - max[1] = max_i; - } - } - return max; - } + /* Track pitches upstream (i.e. forward in time). */ + for (int frameindex = start + 1; frameindex < this.s1.length; frameindex++) { + /* Flag that indicates, if a matching pitch could be found in the new frame. */ + boolean found = false; - /** - * Seeks the pitch with the maximum salience in the specified frame in S1 and returns the - * index i that point to that maximum. If now maximum was found, -1 is returned. - * - * @param t Temporal index of the frame in S1. - * @return Index of the maximum in the specified frame. - */ - private int seekMostSalientInFrameS1(int t) { - Pitch[] pitches = this.s1[t]; - int max = -1; - for (int i=0;i pitches[max].getSalience()) { - max = i; - } + /* Search for a matching pitch candidate in S1 in the next frame. */ + for (int j = 0; j < this.s1[frameindex].length; j++) { + if (this.s1[frameindex][j] == null) { + continue; } - return max; - } + if (found = (Math.abs(this.s1[frameindex][j].distanceCents(lastPitch)) <= this.d_max)) { + lastPitch = this.selectFromS1(frameindex, j); + contour.append(lastPitch); + misses = 0; + break; + } + } + + /* If a pitch candidate was found in S1, continue to next iteration. */ + if (found) { + continue; + } + + /* This code only gets executed if no matching pitch could be found in S1: + * + * Increase number of misses and make sure, that it does not exceed m_max (otherwise return false). + * Search for pitch candidates in S0 afterwards. + */ + misses += 1; + if (misses * this.t_stepsize >= this.m_max) { + break; + } + for (int j = 0; j < this.s0[frameindex].length; j++) { + if (this.s0[frameindex][j] == null) { + continue; + } + if (found = (Math.abs(this.s0[frameindex][j].distanceCents(lastPitch)) <= this.d_max)) { + lastPitch = this.selectFromS0(frameindex, j); + contour.append(lastPitch); + break; + } + } - /** - * Adds a pitch-contour to the list of contours and updates the contour-statistics. - * - * @param contour PitchContour to add. - */ - private void addContour(PitchContour contour) { - this.pitchContours.add(contour); - this.contourStatistics.addValue(contour.salienceMean()); + if (!found) { + break; + } } - /** - * Applies a per-frame filter on pitches in S1 and moves all peaks whose salience is bellow a certain - * threshold from S1 to S0. This filter is described in [1], section II-C. - */ - private void applyPerFrameFilter() { - for (int t = 0; t 0; frameindex--) { + boolean found = false; - private void track(final PitchContour contour, final int start) { - /* If start is the last entry, then no forward-tracking is required. */ - if (start == this.s1.length - 1) { - return; + /* Search for a matching pitch candidate in S1 in the next frame. */ + for (int j = 0; j < this.s1[frameindex].length; j++) { + if (this.s1[frameindex][j] == null) { + continue; } - - /* Initialize helper variables; number of pitches and last-pitch. */ - int misses = 0; - Pitch lastPitch = contour.getPitch(start); - - /* Track pitches upstream (i.e. forward in time). */ - for (int frameindex = start+1; frameindex= this.m_max) { - break; - } - for (int j=0;j= this.m_max) { + break; + } + for (int j = 0; j < this.s0[frameindex].length; j++) { + if (this.s0[frameindex][j] == null) { + continue; } - - /* Re-Initialize helper variables; number of pitches and last-pitch. */ - misses = 0; - lastPitch = contour.getPitch(start); - - /* Track pitches downstream (i.e. back in time) */ - for (int frameindex = start-1; frameindex > 0; frameindex--) { - boolean found = false; - - /* Search for a matching pitch candidate in S1 in the next frame. */ - for (int j=0;j= this.m_max) { - break; - } - for (int j=0;j contours) { - contours.removeIf(c -> { - if (c.pitchDeviation() < 40.0f) { - return c.salienceMean() < contourStatistics.getMean() - this.v_threshold * contourStatistics.getStandardDeviation(); - } else { - return false; - } - }); + /* If no matching pitch was found even in S0, stop tracking. */ + if (!found) { + break; + } } + } + + /** + * + */ + private void voicingDetection(List contours) { + contours.removeIf(c -> { + if (c.pitchDeviation() < 40.0f) { + return c.salienceMean() < contourStatistics.getMean() - this.v_threshold * contourStatistics.getStandardDeviation(); + } else { + return false; + } + }); + } + + private void detectAndRemoveOctaveDuplicates(List contours, double[] meanpitches) { + Iterator iterator = contours.iterator(); + while (iterator.hasNext()) { + PitchContour ct = iterator.next(); + for (PitchContour ci : contours) { + /* Take contour at i; if c == ci then skip. */ + if (ct == ci || ci.overlaps(ct)) { + continue; + } - private void detectAndRemoveOctaveDuplicates(List contours, double[] meanpitches) { - Iterator iterator = contours.iterator(); - while(iterator.hasNext()) { - PitchContour ct = iterator.next(); - for (PitchContour ci : contours) { - /* Take contour at i; if c == ci then skip. */ - if (ct == ci || ci.overlaps(ct)) { - continue; - } - - /* Calculate mean distance from ct to ci. */ - double distance = 0.0; - int count = 0; - for (int k = ct.getStart(); k <= ct.getEnd(); k++) { - if (ci.getPitch(k) != null && ct.getPitch(k) != null) { - distance += ci.getPitch(k).distanceCents(ct.getPitch(k)); - count += 1; - } - } - - /* Normalise distance. */ - distance /= count; - - /* If distance is between 1150 and 1250 cents, the an octave duplicate has been detected! */ - if (distance >= (FrequencyUtils.OCTAVE_CENT - 50.f) && distance <= (FrequencyUtils.OCTAVE_CENT + 50.f)) { - double di = 0.0f; - double dt = 0.0f; - for (int k = ci.getStart(); k <= ci.getEnd(); k++) { - di += Math.abs(ci.getPitch(k).distanceCents((float)meanpitches[k])); - } - for (int k = ct.getStart(); k <= ct.getEnd(); k++) { - dt += Math.abs(ct.getPitch(k).distanceCents((float)meanpitches[k])); - } - - /* Normalise distances. */ - di /= ci.size(); - dt /= ct.size(); - - /* If distance dt > di, then remove ct. */ - if (Math.abs(dt) > Math.abs(di)) { - iterator.remove(); - break; - } - } - } + /* Calculate mean distance from ct to ci. */ + double distance = 0.0; + int count = 0; + for (int k = ct.getStart(); k <= ct.getEnd(); k++) { + if (ci.getPitch(k) != null && ct.getPitch(k) != null) { + distance += ci.getPitch(k).distanceCents(ct.getPitch(k)); + count += 1; + } } - } - private void detectAndRemovePitchOutliers(List contours, final double[] meanpitches) { - Iterator iterator = contours.iterator(); - while(iterator.hasNext()) { - PitchContour c = iterator.next(); - double distance = 0.0f; - for (int t = c.getStart(); t <= c.getEnd(); t++) { - distance += c.getPitch(t).distanceCents((float)meanpitches[t]); - } - distance /= c.size(); - if (Math.abs(distance) > FrequencyUtils.OCTAVE_CENT) { - iterator.remove(); - } + /* Normalise distance. */ + distance /= count; + + /* If distance is between 1150 and 1250 cents, the an octave duplicate has been detected! */ + if (distance >= (FrequencyUtils.OCTAVE_CENT - 50.f) && distance <= (FrequencyUtils.OCTAVE_CENT + 50.f)) { + double di = 0.0f; + double dt = 0.0f; + for (int k = ci.getStart(); k <= ci.getEnd(); k++) { + di += Math.abs(ci.getPitch(k).distanceCents((float) meanpitches[k])); + } + for (int k = ct.getStart(); k <= ct.getEnd(); k++) { + dt += Math.abs(ct.getPitch(k).distanceCents((float) meanpitches[k])); + } + + /* Normalise distances. */ + di /= ci.size(); + dt /= ct.size(); + + /* If distance dt > di, then remove ct. */ + if (Math.abs(dt) > Math.abs(di)) { + iterator.remove(); + break; + } } + } + } + } + + private void detectAndRemovePitchOutliers(List contours, final double[] meanpitches) { + Iterator iterator = contours.iterator(); + while (iterator.hasNext()) { + PitchContour c = iterator.next(); + double distance = 0.0f; + for (int t = c.getStart(); t <= c.getEnd(); t++) { + distance += c.getPitch(t).distanceCents((float) meanpitches[t]); + } + distance /= c.size(); + if (Math.abs(distance) > FrequencyUtils.OCTAVE_CENT) { + iterator.remove(); + } } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetComparator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetComparator.java index 8c395af8c..04b829197 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetComparator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetComparator.java @@ -4,10 +4,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import com.googlecode.javaewah.datastructure.BitSet; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.util.Comparator; import java.util.Map; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; public class BitSetComparator implements Comparator> { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetHammingDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetHammingDistance.java index 34ff11d13..8520f6b34 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetHammingDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BitSetHammingDistance.java @@ -4,7 +4,7 @@ public class BitSetHammingDistance implements Distance { - + @Override public double applyAsDouble(BitSet one, BitSet two) { return one.xorcardinality(two); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BoundedManhattanDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BoundedManhattanDistance.java index f99f0d601..a7f0f6c86 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BoundedManhattanDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/BoundedManhattanDistance.java @@ -6,7 +6,7 @@ public class BoundedManhattanDistance implements FloatArrayDistance { private final double bound; - public BoundedManhattanDistance(double bound){ + public BoundedManhattanDistance(double bound) { this.bound = bound; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/FloatArrayDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/FloatArrayDistance.java index de192619d..375219154 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/FloatArrayDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/FloatArrayDistance.java @@ -7,7 +7,7 @@ public interface FloatArrayDistance extends Distance, DistanceMeasure { static final long serialVersionUID = 1L; - + /** * returns the distance between the two provided arrays or NaN if at least one of them is * null. @@ -15,18 +15,17 @@ public interface FloatArrayDistance extends Distance, DistanceMeasure { @Override double applyAsDouble(float[] t, float[] u); - - + @Override default double compute(double[] a, double[] b) throws DimensionMismatchException { - if(a == null || b == null){ + if (a == null || b == null) { throw new NullPointerException(); } - if(a.length != b.length){ + if (a.length != b.length) { throw new DimensionMismatchException(a.length, b.length); } float[] fa = new float[a.length], fb = new float[b.length]; - for(int i = 0; i < a.length; ++i){ + for (int i = 0; i < a.length; ++i) { fa[i] = (float) a[i]; fb[i] = (float) b[i]; } @@ -34,7 +33,6 @@ default double compute(double[] a, double[] b) throws DimensionMismatchException } - public static FloatArrayDistance fromDistance(final ReadableQueryConfig.Distance distance) { if (distance == null) { throw new NullPointerException("distance cannot be null"); @@ -44,32 +42,32 @@ public static FloatArrayDistance fromDistance(final ReadableQueryConfig.Distance } switch (distance) { - case chebyshev: - return new ChebyshevDistance(); - case chisquared: - return new ChiSquaredDistance(); - case correlation: - return new CorrelationDistance(); - case cosine: - return new CosineDistance(); - case euclidean: - return new EuclideanDistance(); - case hamming: - return new HammingDistance(); - case haversine: - return new HaversineDistance(); - case jaccard: - return new JaccardDistance(); - case kullbackleibler: - return new KLDivergence(); - case manhattan: - return new ManhattanDistance(); - case spannorm: - return new SpanNormDistance(); - case squaredeuclidean: - return new SquaredEuclideanDistance(); - default: - break; + case chebyshev: + return new ChebyshevDistance(); + case chisquared: + return new ChiSquaredDistance(); + case correlation: + return new CorrelationDistance(); + case cosine: + return new CosineDistance(); + case euclidean: + return new EuclideanDistance(); + case hamming: + return new HammingDistance(); + case haversine: + return new HaversineDistance(); + case jaccard: + return new JaccardDistance(); + case kullbackleibler: + return new KLDivergence(); + case manhattan: + return new ManhattanDistance(); + case spannorm: + return new SpanNormDistance(); + case squaredeuclidean: + return new SquaredEuclideanDistance(); + default: + break; } @@ -102,30 +100,30 @@ public static FloatArrayDistance fromQueryConfig(ReadableQueryConfig queryConfig if (queryConfig.getDistanceWeights().isPresent()) { float[] weights = queryConfig.getDistanceWeights().get(); switch (distance) { - case chebyshev: - return new WeightedChebyshevDistance(weights); - case chisquared: - return new WeightedChiSquaredDistance(weights); - case correlation: - return new WeightedCorrelationDistance(weights); - case cosine: - return new WeightedCosineDistance(weights); - case euclidean: - return new WeightedEuclideanDistance(weights); - case hamming: - return new WeightedHammingDistance(weights); - case jaccard: - return new WeightedJaccardDistance(weights); - case kullbackleibler: - return new WeightedKLDivergence(weights); - case manhattan: - return new WeightedManhattanDistance(weights); - case spannorm: - return new WeightedSpanNormDistance(weights); - case squaredeuclidean: - return new WeightedSquaredEuclideanDistance(weights); - default: - throw new IllegalStateException("weighted distance not implemented!"); + case chebyshev: + return new WeightedChebyshevDistance(weights); + case chisquared: + return new WeightedChiSquaredDistance(weights); + case correlation: + return new WeightedCorrelationDistance(weights); + case cosine: + return new WeightedCosineDistance(weights); + case euclidean: + return new WeightedEuclideanDistance(weights); + case hamming: + return new WeightedHammingDistance(weights); + case jaccard: + return new WeightedJaccardDistance(weights); + case kullbackleibler: + return new WeightedKLDivergence(weights); + case manhattan: + return new WeightedManhattanDistance(weights); + case spannorm: + return new WeightedSpanNormDistance(weights); + case squaredeuclidean: + return new WeightedSquaredEuclideanDistance(weights); + default: + throw new IllegalStateException("weighted distance not implemented!"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/HaversineDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/HaversineDistance.java index 039c159a4..c121ee203 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/HaversineDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/HaversineDistance.java @@ -9,11 +9,11 @@ public class HaversineDistance implements FloatArrayDistance { @Override public double applyAsDouble(float[] t, float[] u) { - double dLat = Math.toRadians((u[0] - t[0])); + double dLat = Math.toRadians((u[0] - t[0])); double dLong = Math.toRadians((u[1] - t[1])); double startLat = Math.toRadians(t[0]); - double endLat = Math.toRadians(u[0]); + double endLat = Math.toRadians(u[0]); double a = haversin(dLat) + Math.cos(startLat) * Math.cos(endLat) * haversin(dLong); double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/PrimitiveTypeMapDistanceComparator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/PrimitiveTypeMapDistanceComparator.java index 29f3fc633..a645884bd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/PrimitiveTypeMapDistanceComparator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/PrimitiveTypeMapDistanceComparator.java @@ -3,10 +3,9 @@ import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; - import java.util.Comparator; import java.util.Map; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; public class PrimitiveTypeMapDistanceComparator implements Comparator> { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedChebyshevDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedChebyshevDistance.java index 58a04473d..2674653df 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedChebyshevDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedChebyshevDistance.java @@ -14,26 +14,26 @@ public class WeightedChebyshevDistance extends ChebyshevDistance { } this.weights = weights.clone(); } - + @Override public double applyAsDouble(float[] t, float[] u) { - if(t == null || u == null){ + if (t == null || u == null) { return Double.NaN; } - - if(t == u){ + + if (t == u) { return 0d; } - + int len = Math.min(Math.min(t.length, u.length), this.weights.length); - + double dist = 0d; - - for(int i = 0; i < len; ++i){ + + for (int i = 0; i < len; ++i) { dist = Math.max(dist, Math.abs(t[i] - u[i]) * this.weights[i]); } - + return dist; } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedManhattanDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedManhattanDistance.java index a3adf34e0..172af1d96 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedManhattanDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedManhattanDistance.java @@ -14,7 +14,7 @@ public class WeightedManhattanDistance extends ManhattanDistance { } this.weights = weights.clone(); } - + @Override public double applyAsDouble(float[] t, float[] u) { if (t == null || u == null) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedMinkowskiDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedMinkowskiDistance.java index 8663eb56c..80f8844d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedMinkowskiDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedMinkowskiDistance.java @@ -15,7 +15,7 @@ public class WeightedMinkowskiDistance extends MinkowskiDistance { } this.weights = weights.clone(); } - + @Override public double applyAsDouble(float[] t, float[] u) { if (t == null || u == null) { @@ -36,5 +36,5 @@ public double applyAsDouble(float[] t, float[] u) { return Math.pow(dist, 1d / exponenet); } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSpanNormDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSpanNormDistance.java index ccc8a75b1..3b2d99363 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSpanNormDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSpanNormDistance.java @@ -14,7 +14,7 @@ public class WeightedSpanNormDistance extends SpanNormDistance { } this.weights = weights.clone(); } - + @Override public double applyAsDouble(float[] t, float[] u) { if (t == null || u == null) { @@ -36,5 +36,5 @@ public double applyAsDouble(float[] t, float[] u) { return max - min; } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSquaredEuclideanDistance.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSquaredEuclideanDistance.java index a3b1addda..12b1a05d9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSquaredEuclideanDistance.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/distance/WeightedSquaredEuclideanDistance.java @@ -14,26 +14,26 @@ public class WeightedSquaredEuclideanDistance extends SquaredEuclideanDistance { } this.weights = weights.clone(); } - + @Override public double applyAsDouble(float[] t, float[] u) { - if(t == null || u == null){ + if (t == null || u == null) { return Double.NaN; } - - if(t == u){ + + if (t == u) { return 0d; } - + int len = Math.min(Math.min(t.length, u.length), this.weights.length); - + double dist = 0d; - - for(int i = 0; i < len; ++i){ + + for (int i = 0; i < len; ++i) { dist += (t[i] - u[i]) * (t[i] - u[i]) * this.weights[i]; } - + return dist; } - + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/FrequencyUtils.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/FrequencyUtils.java index dac82bb6c..377661505 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/FrequencyUtils.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/FrequencyUtils.java @@ -3,22 +3,26 @@ public final class FrequencyUtils { - /** */ - private FrequencyUtils() {} + /** + * + */ + private FrequencyUtils() { + } - /** One octave distance in cents. */ - public final static float OCTAVE_CENT = 1200.0f; + /** + * One octave distance in cents. + */ + public final static float OCTAVE_CENT = 1200.0f; - /** - * Returns the distance between two frequencies in Hz on a Cent scale. If the returned - * value is < 0 then f2 < f1, otherwise f2 > f1. - * - * @param f1 First frequency in Hz. - * @param f2 Second frequency in Hz. - * @return Distance in Cents. - */ - public static double cents(float f1, float f2) { - return 1200.0f * (Math.log(f2/f1)/Math.log(2)); - } + /** + * Returns the distance between two frequencies in Hz on a Cent scale. If the returned value is < 0 then f2 < f1, otherwise f2 > f1. + * + * @param f1 First frequency in Hz. + * @param f2 Second frequency in Hz. + * @return Distance in Cents. + */ + public static double cents(float f1, float f2) { + return 1200.0f * (Math.log(f2 / f1) / Math.log(2)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java index 40cf5f2a4..c42c964de 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/SamplingUtilities.java @@ -4,32 +4,31 @@ public class SamplingUtilities { + private SamplingUtilities() { + } + + + /** + * Down-samples a 1D array by the provided downscaling factor and then returns the down-sampled version of the original array. + * + * @param samples Double array that should be down-sampled. + * @param factor Factor by which to down-sample. i.e. factor of n means that every n'th sample is discarded. + */ + public static double[] downsample(double[] samples, int factor) { + /* Makes sure, that the factor is a positive value. */ + factor = Math.abs(factor); + + /* Make sure that array is large enough for down-sampling. */ + if (samples.length < factor) { + throw new IllegalArgumentException(String.format("The provided array of samples (length: %d) is too small to be down-sampled by a factor of %d.", samples.length, factor)); + } - private SamplingUtilities() {} - - - /** - * Down-samples a 1D array by the provided downscaling factor and then returns the - * down-sampled version of the original array. - * - * @param samples Double array that should be down-sampled. - * @param factor Factor by which to down-sample. i.e. factor of n means that every n'th sample is discarded. - */ - public static double[] downsample (double[] samples, int factor) { - /* Makes sure, that the factor is a positive value. */ - factor = Math.abs(factor); - - /* Make sure that array is large enough for down-sampling. */ - if (samples.length < factor) { - throw new IllegalArgumentException(String.format("The provided array of samples (length: %d) is too small to be down-sampled by a factor of %d.", samples.length, factor)); - } - - /* Now downsample. */ - double[] downsampled = new double[(samples.length/factor) - 1]; - int j = 0; - for (int i=factor; i<(samples.length-factor); i+=factor, j+=1) { - downsampled[j] = samples[i]; - } - return downsampled; + /* Now downsample. */ + double[] downsampled = new double[(samples.length / factor) - 1]; + int j = 0; + for (int i = factor; i < (samples.length - factor); i += factor, j += 1) { + downsampled[j] = samples[i]; } + return downsampled; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFT.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFT.java index fec374f85..09de81ece 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFT.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFT.java @@ -16,152 +16,155 @@ *

  • It provides access to some important derivatives of the FFT, like the power-spectrum.
    • *
  • All derivatives are calculated in a lazy way i.e. the values are on access.
    • * - * + *

    * The same instance of the FFT class can be re-used to process multiple samples. Every call to forward() will replace * all the existing data in the instance. - * + *

    * The inspiration for this class comes from the FFT class found in the jAudio framework (see * https://github.com/dmcennis/jaudioGIT) * * @see WindowFunction - * */ public class FFT { - /** Data obtained by forward FFT. */ - private Complex[] data; - - /** Magnitude spectrum of the FFT data. May be null if it has not been obtained yet. */ - private Spectrum magnitudeSpectrum; - - /** Power spectrum of the FFT data. May be null if it has not been obtained yet. */ - private Spectrum powerSpectrum; - - /** WindowFunction to apply before forward transformation. Defaults to IdentityWindows (= no window). */ - private WindowFunction windowFunction = new RectangularWindow(); - - /** Samplingrate of the last chunk of data that was processed by FFT. */ - private float samplingrate; - - /** - * Performs a forward fourier transformation on the provided, real valued data. The method makes sure, - * that the size of the array is a power of two (for which the FFT class has been optimized) and pads - * the data with zeros if necessary. Furthermore, one can provide a WindowingFunction that will be applied - * on the data. - * - * Important: Every call to forward() replaces all the existing data in the current instance. I.e. - * the same instance of FFT can be re-used. - * - * @param data Data to be transformed. - * @param window WindowFunction to use for the transformation. - */ - public void forward(double[] data, float samplingrate, WindowFunction window) { - this.windowFunction = window; - this.samplingrate = samplingrate; - - int actual_length = data.length; - int valid_length = FFTUtil.nextPowerOf2(actual_length); - double[] reshaped = new double[valid_length]; - for (int i = 0; iImportant: Every call to forward() replaces all the existing data in the current instance. I.e. + * the same instance of FFT can be re-used. + * + * @param data Data to be transformed. + * @param window WindowFunction to use for the transformation. + */ + public void forward(double[] data, float samplingrate, WindowFunction window) { + this.windowFunction = window; + this.samplingrate = samplingrate; + + int actual_length = data.length; + int valid_length = FFTUtil.nextPowerOf2(actual_length); + double[] reshaped = new double[valid_length]; + for (int i = 0; i < reshaped.length; i++) { + if (i < actual_length) { + reshaped[i] = data[i] * this.windowFunction.value(i, valid_length); + } else { + reshaped[i] = 0; + } } - /** - * Returns the power spectrum of the transformed data. If that spectrum has not been - * calculated yet it will be upon invocation of the method. - * - * @return Array containing the power for each frequency bin. - */ - public Spectrum getPowerSpectrum() { - if (this.powerSpectrum == null) { - this.powerSpectrum = Spectrum.createPowerSpectrum(this.data, this.samplingrate, this.windowFunction); - } - return this.powerSpectrum; + /* Perform FFT using FastFourierTransformer library. */ + FastFourierTransformer transformer = new FastFourierTransformer(DftNormalization.STANDARD); + this.data = transformer.transform(reshaped, TransformType.FORWARD); + + /* Reset the calculated properties. */ + this.powerSpectrum = null; + this.magnitudeSpectrum = null; + } + + /** + * Returns the magnitude spectrum of the transformed data. If that spectrum has not been calculated yet it will be upon invocation of the method. + * + * @return Array containing the magnitude for each frequency bin. + */ + public Spectrum getMagnitudeSpectrum() { + if (this.magnitudeSpectrum == null) { + this.magnitudeSpectrum = Spectrum.createMagnitudeSpectrum(this.data, this.samplingrate, this.windowFunction); } - /** - * Getter for the transformed data. - * - * @return Array containing the raw FFT data. - */ - public final Complex[] getValues() { - return this.data; + return this.magnitudeSpectrum; + } + + /** + * Returns the power spectrum of the transformed data. If that spectrum has not been calculated yet it will be upon invocation of the method. + * + * @return Array containing the power for each frequency bin. + */ + public Spectrum getPowerSpectrum() { + if (this.powerSpectrum == null) { + this.powerSpectrum = Spectrum.createPowerSpectrum(this.data, this.samplingrate, this.windowFunction); } - - /** - * Can be used to directly access a FFT coefficient at the - * specified index. - * - * @param index Index of the coefficient that should be retrieved. - * @return Fourier coefficient. - */ - public final Complex get(int index) { - return this.data[index]; - } - - /** - * Getter for samplingrate. - * - * @return Rate at which the original signal has been sampled. - */ - public final float getSamplingrate() { - return this.samplingrate; - } - - /** - * Getter for samplingrate. - * - * @return Rate at which the original signal has been sampled. - */ - public final int getWindowsize() { - return this.data.length; - } - - /** - * Returns true if the FFT only contains zeros and false - * otherwise - */ - public final boolean isZero() { - for (Complex coefficient : this.data) { - if (coefficient.abs() > 0) { - return false; - } - } - return true; - } - - /** - * Applies the provided FrequencyDomainFilter to this FFT. - * - * @param filter FrequencyDomainFilter that should be applied. - */ - public final void applyFilter(FrequencyDomainFilterInterface filter) { - filter.filterInPlace(this.data); + return this.powerSpectrum; + } + + /** + * Getter for the transformed data. + * + * @return Array containing the raw FFT data. + */ + public final Complex[] getValues() { + return this.data; + } + + /** + * Can be used to directly access a FFT coefficient at the specified index. + * + * @param index Index of the coefficient that should be retrieved. + * @return Fourier coefficient. + */ + public final Complex get(int index) { + return this.data[index]; + } + + /** + * Getter for samplingrate. + * + * @return Rate at which the original signal has been sampled. + */ + public final float getSamplingrate() { + return this.samplingrate; + } + + /** + * Getter for samplingrate. + * + * @return Rate at which the original signal has been sampled. + */ + public final int getWindowsize() { + return this.data.length; + } + + /** + * Returns true if the FFT only contains zeros and false otherwise + */ + public final boolean isZero() { + for (Complex coefficient : this.data) { + if (coefficient.abs() > 0) { + return false; + } } + return true; + } + + /** + * Applies the provided FrequencyDomainFilter to this FFT. + * + * @param filter FrequencyDomainFilter that should be applied. + */ + public final void applyFilter(FrequencyDomainFilterInterface filter) { + filter.filterInPlace(this.data); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java index eb506abe9..24486677b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/FFTUtil.java @@ -4,133 +4,123 @@ /** * Some utility functions in the context of FFT and STFT. - * */ public class FFTUtil { - private FFTUtil() { } + private FFTUtil() { + } - /** - * Returns frequency labels in Hz for a FFT of the specified size and samplingrate. - * - * @param size Size of the FFT (i.e. number of frequency bins). - * @param samplingrate Rate at which the original data has been sampled. - * - * @return Array containing the frequency labels in Hz in ascending order. - */ - public static float[] binCenterFrequencies(int size, float samplingrate) { - float[] labels = new float[size/2]; - for (int bin = 0; bin < labels.length; bin++) { - labels[bin] = FFTUtil.binCenterFrequency(bin, size, samplingrate); - } - return labels; + /** + * Returns frequency labels in Hz for a FFT of the specified size and samplingrate. + * + * @param size Size of the FFT (i.e. number of frequency bins). + * @param samplingrate Rate at which the original data has been sampled. + * @return Array containing the frequency labels in Hz in ascending order. + */ + public static float[] binCenterFrequencies(int size, float samplingrate) { + float[] labels = new float[size / 2]; + for (int bin = 0; bin < labels.length; bin++) { + labels[bin] = FFTUtil.binCenterFrequency(bin, size, samplingrate); } + return labels; + } - /** - * Returns the center frequency associated with the provided bin-index for the given - * window-size and samplingrate. - * - * @param index Index of the bin in question. - * @param size Size of the FFT (i.e. number of frequency bins). - * @param samplingrate Rate at which the original data has been sampled. - */ - public static float binCenterFrequency(int index, int size, float samplingrate) { - if (index > size) { - throw new IllegalArgumentException("The index cannot be greater than the window-size of the FFT."); - } - double bin_width = (samplingrate / size); - double offset = bin_width/2.0; - return (float)((index * bin_width) + offset); + /** + * Returns the center frequency associated with the provided bin-index for the given window-size and samplingrate. + * + * @param index Index of the bin in question. + * @param size Size of the FFT (i.e. number of frequency bins). + * @param samplingrate Rate at which the original data has been sampled. + */ + public static float binCenterFrequency(int index, int size, float samplingrate) { + if (index > size) { + throw new IllegalArgumentException("The index cannot be greater than the window-size of the FFT."); } + double bin_width = (samplingrate / size); + double offset = bin_width / 2.0; + return (float) ((index * bin_width) + offset); + } - /** - * Returns the bin-index associated with the provided frequency at the given samplingrate - * and window-size. - * - * @param size Size of the FFT (i.e. number of frequency bins). - * @param samplingrate Rate at which the original data has been sampled. - */ - public static int binIndex(float frequency, int size, float samplingrate) { - if (frequency > samplingrate/2) { - throw new IllegalArgumentException("The frequency cannot be greater than half the samplingrate."); - } - double bin_width = (samplingrate / size); - return (int)Math.floor(frequency/bin_width); + /** + * Returns the bin-index associated with the provided frequency at the given samplingrate and window-size. + * + * @param size Size of the FFT (i.e. number of frequency bins). + * @param samplingrate Rate at which the original data has been sampled. + */ + public static int binIndex(float frequency, int size, float samplingrate) { + if (frequency > samplingrate / 2) { + throw new IllegalArgumentException("The frequency cannot be greater than half the samplingrate."); } + double bin_width = (samplingrate / size); + return (int) Math.floor(frequency / bin_width); + } - /** - * Returns time labels in seconds for a STFT of given width using the provided - * window size and samplerate. - * - * @param width Size of the STFT (i.e. number of time bins). - * @param windowsize Used for FFT (i.e. number of samples per time bin) - * @param overlap Overlap in samples between two adjacent windows during the FFT. - * @param padding Zeropadding, i.e. how many zeros have been added before and after the actual sample starts - * (Assumption: padding happens within the fixed windowsize) - * @param samplingrate Rate at which the original data has been sampled. - * - * @return Array containing the time labels for the STFT in seconds in ascending order. - */ - public static float[] time(int width, int windowsize, int overlap, int padding, float samplingrate) { - float[] labels = new float[width]; - float stepsize = FFTUtil.timeStepsize(windowsize, overlap, padding, samplingrate); - for (int i=0;i parametersForDuration(float samplingrate, float windowduration){ - int samples = (int)(samplingrate * windowduration); - int windowsize = nextPowerOf2(samples); - return new Pair<>(windowsize,(windowsize-samples)/2); - } + /** + * Calculates and returns the windowsize in samples and the zero-padding in samples so as to achieve a certain window-duration in seconds. + * + * @param samplingrate The samplingrate of the original data. + * @param windowduration Duration of the window in seconds. + * @return Pair of integers; first integer determines the windowsize and the second determines the padding. + */ + public static Pair parametersForDuration(float samplingrate, float windowduration) { + int samples = (int) (samplingrate * windowduration); + int windowsize = nextPowerOf2(samples); + return new Pair<>(windowsize, (windowsize - samples) / 2); + } - /** - * Checks if the provided number is a power of two and return true if so and false otherwise. - * - * @param number Number to check. - * @return true if number is a power of two, false otherwise. - */ - public static boolean isPowerOf2(int number) { - double value = Math.log(number)/Math.log(2); - return Math.ceil(value) == value; - } + /** + * Checks if the provided number is a power of two and return true if so and false otherwise. + * + * @param number Number to check. + * @return true if number is a power of two, false otherwise. + */ + public static boolean isPowerOf2(int number) { + double value = Math.log(number) / Math.log(2); + return Math.ceil(value) == value; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java index c47d01228..e481a54d0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/STFT.java @@ -1,242 +1,253 @@ package org.vitrivr.cineast.core.util.dsp.fft; -import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import org.vitrivr.cineast.core.util.dsp.filter.frequency.FrequencyDomainFilterInterface; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; +import org.vitrivr.cineast.core.util.dsp.filter.frequency.FrequencyDomainFilterInterface; public class STFT { - /** Size of the FFT window. Must be a power of 2 (e.g. 512, 1024, 2048, 4096). */ - private final int windowsize; - - /** Sampling rate with which the samples have been sampled. */ - private final float samplingrate; - - /** Window function to use when calculating the FFT. */ - private final WindowFunction windowFunction; - - /** - * Zero-padding factor used for STFT calculation. - * - * A value of X means that X zeros will be prepended before the signal start. Then after the actual signal, another X zeros - * will be appended. This means, that the window will contain windowsize-2*X samplepoints - */ - private final int padding; - - /** Overlap in samples between two subsequent windows. */ - private final int overlap; - - /** Height of the STFT (i.e. the number of frequency bins per FFT). */ - private int height; - - /** Frequency labels in ascending order (for all FFTs). */ - private final float[] frequencies; - - /** Time labels in ascending order (for each entry in the stft list). */ - private float[] time; - - /** List containing one FFT entry per timepoint. Same order as time[] */ - private final List stft; - - public STFT(int windowsize, int overlap, int padding, WindowFunction function, float samplingrate) { - /* Make sure that the windowsize is a power of two. */ - if (!FFTUtil.isPowerOf2(windowsize)) { - throw new IllegalArgumentException("The provided window size of " + windowsize + " is not a power of two!"); - } - - /* Store the local variables. */ - this.samplingrate = samplingrate; - - /* Store the window size and window function. */ - this.windowFunction = function; - this.windowsize = windowsize; - this.overlap = overlap; - this.padding = padding; - this.height = windowsize/2; - /* Prepares empty array for STFT. */ - this.stft = new ArrayList<>(); - - this.frequencies = FFTUtil.binCenterFrequencies(windowsize, samplingrate); - } - - /** - * Performs a forward fourier transformation on the provided, real valued data and appends it to the FFT. The caller - * of this function must make sure, that the data conforms to the properties specified upon construction of this class. - * Otherwise, some results may be unexpected! - * - * Important: Every call to forward() appends a time-local DFT to the current STFT. The existing - * data will be kept. - */ - public void forward(double[] samples) { - /* Initialize values for the sliding window. */ - final int increment = this.windowsize - overlap - 2*this.padding; - final int length = this.windowsize - 2*this.padding; - int start = 0; - int end = start + length - 1; - - /* Initialize buffer that holds samples for FFT. */ - final double window[] = new double[windowsize]; - - /* - * Outer-loop: Create a sliding window and move it across the samples. - * For each iteration, forward the FFT for the samples in the Window and add it to the list. - */ - - while (start < samples.length) { - /* Copy the samples into the window. */ - if (end < samples.length) { - System.arraycopy(samples, start, window, padding, length); - } else { - System.arraycopy(samples, start, window, padding, samples.length - start); - Arrays.fill(window, samples.length - start + 1, window.length - 1, 0.0); - } - - /* Create Forward FFT entries for each window. */ - FFT fft = new FFT(); - fft.forward(window, this.samplingrate, this.windowFunction); - this.stft.add(fft); - - /* Move the window. */ - start += increment; - end += increment; - } - - /* Updates the time-labels. */ - this.time = FFTUtil.time(this.stft.size(), windowsize, overlap, padding, samplingrate); - } - - /** - * Assembles a list of power-spectra (one per FFT contained in this STFT) - * and returns it. - * - * @return List of power-spectra - */ - public List getPowerSpectrum() { - List spectrum = new ArrayList<>(this.stft.size()); - for (FFT fft : this.stft) { - spectrum.add(fft.getPowerSpectrum()); - } - return spectrum; - } - - /** - * Assembles a list of magnitude-spectra (one per FFT contained in this STFT) - * and returns it. - * - * @return List of magnitude-spectra - */ - public List getMagnitudeSpectrum() { - List spectrum = new ArrayList<>(this.stft.size()); - for (FFT fft : this.stft) { - spectrum.add(fft.getMagnitudeSpectrum()); - } - return spectrum; - } - - /** - * Getter for frequency bin labels. - */ - public final float[] getFrequencies() { - return frequencies; - } - - /** - * Returns the number of frequency-bins. - */ - public final int getNumberOfBins() { - return this.frequencies.length; - } - - /** - * Returns the size / width of an individual frequency bin. - */ - public final float getBinSize() { - return (this.samplingrate/this.windowsize); - } - - /** - * Getter for time labels. - */ - public final float[] getTime() { - return time; - } - - /** - * Getter for STFT. - */ - public final List getStft() { - return Collections.unmodifiableList(this.stft); - } - - /** - * Getter for window-size. - */ - public final int getWindowsize() { - return this.windowsize; - } - - /** - * Getter for overlap value. - */ - public int getOverlap() { - return this.overlap; - } - - /** - * Getter for sampling rate. - */ - public final float getSamplingrate() { - return this.samplingrate; - } - - /** - * Getter for the WindowFunction. - * - * @return WindowFunction object. - */ - public WindowFunction getWindowFunction() { - return windowFunction; - } - - /** - * Getter for width of the STFT (i.e. number of timepoints). - * - * @return Width of the STFT. - */ - public int getWidth() { - return this.stft.size(); - } - - /** - * Getter for height of the STFT (i.e. number of frequency bins). - * - * @return Height of the STFT. - */ - public int getHeight() { - return this.height; - } - - /** - * The stepsize in seconds between to adjacent bins in the time dimension. - */ - public float timeStepsize() { - return ((windowsize - overlap - 2*padding)/this.samplingrate); - } - - /** - * Applies the provided FrequencyDomainFilter to this FFT. - * - * @param filter FrequencyDomainFilter that should be applied. - */ - public final void applyFilter(FrequencyDomainFilterInterface filter) { - for (FFT fft : this.getStft()) { - fft.applyFilter(filter); - } - } + /** + * Size of the FFT window. Must be a power of 2 (e.g. 512, 1024, 2048, 4096). + */ + private final int windowsize; + + /** + * Sampling rate with which the samples have been sampled. + */ + private final float samplingrate; + + /** + * Window function to use when calculating the FFT. + */ + private final WindowFunction windowFunction; + + /** + * Zero-padding factor used for STFT calculation. + *

    + * A value of X means that X zeros will be prepended before the signal start. Then after the actual signal, another X zeros will be appended. This means, that the window will contain windowsize-2*X samplepoints + */ + private final int padding; + + /** + * Overlap in samples between two subsequent windows. + */ + private final int overlap; + + /** + * Height of the STFT (i.e. the number of frequency bins per FFT). + */ + private int height; + + /** + * Frequency labels in ascending order (for all FFTs). + */ + private final float[] frequencies; + + /** + * Time labels in ascending order (for each entry in the stft list). + */ + private float[] time; + + /** + * List containing one FFT entry per timepoint. Same order as time[] + */ + private final List stft; + + public STFT(int windowsize, int overlap, int padding, WindowFunction function, float samplingrate) { + /* Make sure that the windowsize is a power of two. */ + if (!FFTUtil.isPowerOf2(windowsize)) { + throw new IllegalArgumentException("The provided window size of " + windowsize + " is not a power of two!"); + } + + /* Store the local variables. */ + this.samplingrate = samplingrate; + + /* Store the window size and window function. */ + this.windowFunction = function; + this.windowsize = windowsize; + this.overlap = overlap; + this.padding = padding; + this.height = windowsize / 2; + + /* Prepares empty array for STFT. */ + this.stft = new ArrayList<>(); + + this.frequencies = FFTUtil.binCenterFrequencies(windowsize, samplingrate); + } + + /** + * Performs a forward fourier transformation on the provided, real valued data and appends it to the FFT. The caller of this function must make sure, that the data conforms to the properties specified upon construction of this class. Otherwise, some results may be unexpected! + * + * Important: Every call to forward() appends a time-local DFT to the current STFT. The existing + * data will be kept. + */ + public void forward(double[] samples) { + /* Initialize values for the sliding window. */ + final int increment = this.windowsize - overlap - 2 * this.padding; + final int length = this.windowsize - 2 * this.padding; + int start = 0; + int end = start + length - 1; + + /* Initialize buffer that holds samples for FFT. */ + final double window[] = new double[windowsize]; + + /* + * Outer-loop: Create a sliding window and move it across the samples. + * For each iteration, forward the FFT for the samples in the Window and add it to the list. + */ + + while (start < samples.length) { + /* Copy the samples into the window. */ + if (end < samples.length) { + System.arraycopy(samples, start, window, padding, length); + } else { + System.arraycopy(samples, start, window, padding, samples.length - start); + Arrays.fill(window, samples.length - start + 1, window.length - 1, 0.0); + } + + /* Create Forward FFT entries for each window. */ + FFT fft = new FFT(); + fft.forward(window, this.samplingrate, this.windowFunction); + this.stft.add(fft); + + /* Move the window. */ + start += increment; + end += increment; + } + + /* Updates the time-labels. */ + this.time = FFTUtil.time(this.stft.size(), windowsize, overlap, padding, samplingrate); + } + + /** + * Assembles a list of power-spectra (one per FFT contained in this STFT) and returns it. + * + * @return List of power-spectra + */ + public List getPowerSpectrum() { + List spectrum = new ArrayList<>(this.stft.size()); + for (FFT fft : this.stft) { + spectrum.add(fft.getPowerSpectrum()); + } + return spectrum; + } + + /** + * Assembles a list of magnitude-spectra (one per FFT contained in this STFT) and returns it. + * + * @return List of magnitude-spectra + */ + public List getMagnitudeSpectrum() { + List spectrum = new ArrayList<>(this.stft.size()); + for (FFT fft : this.stft) { + spectrum.add(fft.getMagnitudeSpectrum()); + } + return spectrum; + } + + /** + * Getter for frequency bin labels. + */ + public final float[] getFrequencies() { + return frequencies; + } + + /** + * Returns the number of frequency-bins. + */ + public final int getNumberOfBins() { + return this.frequencies.length; + } + + /** + * Returns the size / width of an individual frequency bin. + */ + public final float getBinSize() { + return (this.samplingrate / this.windowsize); + } + + /** + * Getter for time labels. + */ + public final float[] getTime() { + return time; + } + + /** + * Getter for STFT. + */ + public final List getStft() { + return Collections.unmodifiableList(this.stft); + } + + /** + * Getter for window-size. + */ + public final int getWindowsize() { + return this.windowsize; + } + + /** + * Getter for overlap value. + */ + public int getOverlap() { + return this.overlap; + } + + /** + * Getter for sampling rate. + */ + public final float getSamplingrate() { + return this.samplingrate; + } + + /** + * Getter for the WindowFunction. + * + * @return WindowFunction object. + */ + public WindowFunction getWindowFunction() { + return windowFunction; + } + + /** + * Getter for width of the STFT (i.e. number of timepoints). + * + * @return Width of the STFT. + */ + public int getWidth() { + return this.stft.size(); + } + + /** + * Getter for height of the STFT (i.e. number of frequency bins). + * + * @return Height of the STFT. + */ + public int getHeight() { + return this.height; + } + + /** + * The stepsize in seconds between to adjacent bins in the time dimension. + */ + public float timeStepsize() { + return ((windowsize - overlap - 2 * padding) / this.samplingrate); + } + + /** + * Applies the provided FrequencyDomainFilter to this FFT. + * + * @param filter FrequencyDomainFilter that should be applied. + */ + public final void applyFilter(FrequencyDomainFilterInterface filter) { + for (FFT fft : this.getStft()) { + fft.applyFilter(filter); + } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java index 24eb03565..ddc6e9431 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/Spectrum.java @@ -1,319 +1,326 @@ package org.vitrivr.cineast.core.util.dsp.fft; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import org.apache.commons.math3.complex.Complex; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.commons.math3.util.MathArrays; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -public class Spectrum implements Iterable>{ - /** - * Available type of spectra. - */ - public enum Type { - POWER, MAGNITUDE; +public class Spectrum implements Iterable> { + + /** + * Available type of spectra. + */ + public enum Type { + POWER, MAGNITUDE; + } + + /** + * Values of the spectrum. They depend on the kind of spectrum being used. + */ + private final double[] spectrum; + + /** + * Frequency labels of the spectrum (usually the x-axis). + */ + private final float[] frequencies; + + /** + * Length of the spectrum (i.e. the number of bins). + */ + private final int length; + + /** + * Type of spectrum as defined in the Type enum. + */ + private final Type type; + + /** + * + */ + private Integer maximumIndex; + + /** + * + */ + private Integer minimumIndex; + + /** + * Calculates and returns the power spectrum for the provided, complex data input (usually stemming from a FFT). + * + * @param data Array of complex values to forward the spectrum from. + * @param samplingrate Sampling rate at which the original samples were obtained. + * @param windowFunction Window function that was used when calculating the FFT. + * @return A power spectrum. + */ + public static Spectrum createPowerSpectrum(Complex[] data, float samplingrate, WindowFunction windowFunction) { + int bins = data.length / 2; + double normalizationFactor = data.length * windowFunction.normalization(data.length); + + double[] powerSpectrum = new double[bins]; + powerSpectrum[0] = (Math.pow(data[0].getReal(), 2) + Math.pow(data[0].getImaginary(), 2)) / normalizationFactor; + for (int i = 1; i < bins; i++) { + powerSpectrum[i] = (2 * Math.pow(data[i].abs(), 2)) / normalizationFactor; } - /** Values of the spectrum. They depend on the kind of spectrum being used. */ - private final double[] spectrum; - - /** Frequency labels of the spectrum (usually the x-axis). */ - private final float[] frequencies; - - /** Length of the spectrum (i.e. the number of bins). */ - private final int length; - - /** Type of spectrum as defined in the Type enum. */ - private final Type type; - - /** */ - private Integer maximumIndex; - - /** */ - private Integer minimumIndex; - - /** - * Calculates and returns the power spectrum for the provided, complex data input (usually - * stemming from a FFT). - * - * @param data Array of complex values to forward the spectrum from. - * @param samplingrate Sampling rate at which the original samples were obtained. - * @param windowFunction Window function that was used when calculating the FFT. - * @return A power spectrum. - */ - public static Spectrum createPowerSpectrum(Complex[] data, float samplingrate, WindowFunction windowFunction) { - int bins = data.length / 2; - double normalizationFactor = data.length * windowFunction.normalization(data.length); - - double[] powerSpectrum = new double[bins]; - powerSpectrum[0] = (Math.pow(data[0].getReal(),2) + Math.pow(data[0].getImaginary(),2)) / normalizationFactor; - for(int i = 1; i < bins; i++) { - powerSpectrum[i] = (2 * Math.pow(data[i].abs(),2)) / normalizationFactor; - } - - return new Spectrum(Type.POWER, powerSpectrum, FFTUtil.binCenterFrequencies(data.length, samplingrate)); + return new Spectrum(Type.POWER, powerSpectrum, FFTUtil.binCenterFrequencies(data.length, samplingrate)); + } + + /** + * Calculates and returns t magnitude spectrum for the provided, complex data input (usually stemming from a FFT). + * + * @param data Array of complex values to forward the spectrum from. + * @param samplingrate Samplingrate at which the original samples were obtained. + * @param windowFunction Window function that was used when calculating the FFT. + * @return A magnitude spectrum. + */ + public static Spectrum createMagnitudeSpectrum(Complex[] data, float samplingrate, WindowFunction windowFunction) { + int bins = data.length / 2; + double normalizationFactor = data.length * windowFunction.normalization(data.length); + + double[] magnitudeSpectrum = new double[bins]; + magnitudeSpectrum[0] = Math.sqrt(Math.pow(data[0].getReal(), 2) + Math.pow(data[0].getImaginary(), 2)) / normalizationFactor; + for (int i = 1; i < bins; i++) { + magnitudeSpectrum[i] = (2 * data[i].abs()) / normalizationFactor; } - /** - * Calculates and returns t magnitude spectrum for the provided, complex data input (usually - * stemming from a FFT). - * - * @param data Array of complex values to forward the spectrum from. - * @param samplingrate Samplingrate at which the original samples were obtained. - * @param windowFunction Window function that was used when calculating the FFT. - * @return A magnitude spectrum. - */ - public static Spectrum createMagnitudeSpectrum(Complex[] data, float samplingrate, WindowFunction windowFunction) { - int bins = data.length / 2; - double normalizationFactor = data.length * windowFunction.normalization(data.length); - - double[] magnitudeSpectrum = new double[bins]; - magnitudeSpectrum[0] = Math.sqrt(Math.pow(data[0].getReal(),2) + Math.pow(data[0].getImaginary(),2)) / normalizationFactor; - for(int i = 1; i < bins; i++) { - magnitudeSpectrum[i] = (2 * data[i].abs()) / normalizationFactor; - } + return new Spectrum(Type.MAGNITUDE, magnitudeSpectrum, FFTUtil.binCenterFrequencies(data.length, samplingrate)); + } - return new Spectrum(Type.MAGNITUDE, magnitudeSpectrum, FFTUtil.binCenterFrequencies(data.length, samplingrate)); + public Spectrum(Type type, double[] spectrum, float[] frequencies) { + /* Check if the length of both array is the same. */ + if (spectrum.length != frequencies.length) { + throw new IllegalArgumentException("The length of the spectrum and the frequency-label array must be the same!"); } - public Spectrum(Type type, double[] spectrum, float[] frequencies) { - /* Check if the length of both array is the same. */ - if (spectrum.length != frequencies.length) { - throw new IllegalArgumentException("The length of the spectrum and the frequency-label array must be the same!"); - } - - /* Store values for further reference. */ - this.length = spectrum.length; - this.spectrum = spectrum; - this.frequencies = frequencies; - this.type = type; + /* Store values for further reference. */ + this.length = spectrum.length; + this.spectrum = spectrum; + this.frequencies = frequencies; + this.type = type; + } + + /** + * Getter for the type of spectrum. + * + * @return Type of the spectrum. + */ + public Type getType() { + return type; + } + + /** + * Returns the size of the spectrum (i.e. the number of bins). + * + * @return Size of the spectrum. + */ + public int size() { + return length; + } + + /** + * Returns a pair of frequency and the associated value at the specified index. + * + * @param idx Index, zero-based and smaller than this.length + * @return Frequency at specified index. + */ + public Pair get(int idx) { + if (idx < this.frequencies.length && idx < this.spectrum.length) { + return new Pair<>(this.frequencies[idx], this.spectrum[idx]); + } else { + return null; } - - /** - * Getter for the type of spectrum. - * - * @return Type of the spectrum. - */ - public Type getType() { - return type; + } + + /** + * Returns the frequency at the specified index. + * + * @param idx Index, zero-based and smaller than this.length + * @return Frequency at specified index. + */ + public Float getFrequency(int idx) { + if (idx < this.length) { + return this.frequencies[idx]; + } else { + return null; } - - /** - * Returns the size of the spectrum (i.e. the number of bins). - * - * @return Size of the spectrum. - */ - public int size() { - return length; + } + + /** + * Returns the value at the specified index. + * + * @param idx Index, zero-based and smaller than this.length + * @return Value at specified index. + */ + public Double getValue(int idx) { + if (idx < this.length) { + return this.spectrum[idx]; + } else { + return null; } - - /** - * Returns a pair of frequency and the associated value at the specified - * index. - * - * @param idx Index, zero-based and smaller than this.length - * @return Frequency at specified index. - */ - public Pair get(int idx) { - if (idx < this.frequencies.length && idx < this.spectrum.length) { - return new Pair<>(this.frequencies[idx], this.spectrum[idx]); - } else { - return null; - } + } + + /** + * Changes the value at the given index to the new, provided value. + * + * @param idx Index of the position in the spectrum that should be updated. + * @param value New value at that position. + */ + public void setValue(int idx, double value) { + if (idx < this.length) { + this.spectrum[idx] = value; } - - /** - * Returns the frequency at the specified index. - * - * @param idx Index, zero-based and smaller than this.length - * @return Frequency at specified index. - */ - public Float getFrequency(int idx) { - if (idx < this.length) { - return this.frequencies[idx]; - } else { - return null; + } + + /** + * Returns the maximum value in the spectrum. + */ + public Pair getMaximum() { + if (this.maximumIndex == null) { + this.maximumIndex = 0; + for (int i = 0; i < this.spectrum.length; i++) { + if (this.spectrum[i] > this.spectrum[this.maximumIndex]) { + this.maximumIndex = i; } + } } - - /** - * Returns the value at the specified index. - * - * @param idx Index, zero-based and smaller than this.length - * @return Value at specified index. - */ - public Double getValue(int idx) { - if (idx < this.length) { - return this.spectrum[idx]; - } else { - return null; + return this.get(this.maximumIndex); + } + + /** + * Returns the minimum value in the spectrum. + */ + public Pair getMinimum() { + if (this.minimumIndex == null) { + this.minimumIndex = 0; + for (int i = 0; i < this.spectrum.length; i++) { + if (this.spectrum[i] < this.spectrum[this.minimumIndex]) { + this.minimumIndex = i; } + } } - - /** - * Changes the value at the given index to the new, - * provided value. - * - * @param idx Index of the position in the spectrum that should be updated. - * @param value New value at that position. - */ - public void setValue(int idx, double value) { - if (idx < this.length) { - this.spectrum[idx] = value; - } + return this.get(this.minimumIndex); + } + + /** + * + */ + public void normalize() { + MathArrays.scaleInPlace(1.0 / this.getMaximum().second, this.spectrum); + } + + /** + * Returns the double array that holds the spectrum data. + * + * @return Double array with spectrum data. + */ + public double[] array() { + return this.spectrum; + } + + /** + * Returns a reduced version of the spectrum, limiting the view to the specified frequency-range. + * + * @param minFreq Minimum frequency to consider. Frequencies < minFreq will be discarded. + * @param maxFreq Maximum frequency to consider. Frequencies > maxFreq will be discarded. + * @return Reduced spectrum. + */ + public Spectrum reduced(float minFreq, float maxFreq) { + if (minFreq >= maxFreq) { + throw new IllegalArgumentException("Minimum frequency must be smaller than maximum frequency!"); } - /** - * Returns the maximum value in the spectrum. - */ - public Pair getMaximum() { - if (this.maximumIndex == null) { - this.maximumIndex = 0; - for (int i=0;i this.spectrum[this.maximumIndex]) { - this.maximumIndex = i; - } - } - } - return this.get(this.maximumIndex); - } + int[] range = new int[2]; - /** - * Returns the minimum value in the spectrum. - */ - public Pair getMinimum() { - if (this.minimumIndex == null) { - this.minimumIndex = 0; - for (int i=0;i maxFreq) { + break; + } } - /** - * Returns the double array that holds the spectrum data. - * - * @return Double array with spectrum data. - */ - public double[] array() { - return this.spectrum; + return new Spectrum(this.type, Arrays.copyOfRange(this.spectrum, range[0], range[1]), Arrays.copyOfRange(this.frequencies, range[0], range[1])); + } + + /** + * Find local maxima in the spectrum and returns the indices of those maxima as integer array. + * + * @param threshold Threshold for search. Values bellow that threshold won't be considered. + * @return Array containing indices (zero-based) of local maxima. + */ + public List> findLocalMaxima(double threshold, boolean significant) { + List> peaks = new ArrayList<>(); + for (int i = 1; i < this.spectrum.length - 1; i++) { + if (this.spectrum[i] < threshold) { + continue; + } + if (spectrum[i] > Math.max(spectrum[i + 1], spectrum[i - 1])) { + peaks.add(this.get(i)); + } } - /** - * Returns a reduced version of the spectrum, limiting the view to the specified frequency-range. - * - * @param minFreq Minimum frequency to consider. Frequencies < minFreq will be discarded. - * @param maxFreq Maximum frequency to consider. Frequencies > maxFreq will be discarded. - * @return Reduced spectrum. - */ - public Spectrum reduced(float minFreq, float maxFreq) { - if (minFreq >= maxFreq) { - throw new IllegalArgumentException("Minimum frequency must be smaller than maximum frequency!"); - } - - int[] range = new int[2]; - - for (int i=0; i < this.frequencies.length; i++) { - if (this.frequencies[i] <= minFreq) { - range[0] = i; - } - if (this.frequencies[i] < maxFreq) { - range[1] = i; - } - - if (this.frequencies[i] > maxFreq) { - break; - } - } - - return new Spectrum(this.type, Arrays.copyOfRange(this.spectrum, range[0], range[1]), Arrays.copyOfRange(this.frequencies, range[0], range[1])); + if (significant) { + DescriptiveStatistics statistics = new DescriptiveStatistics(); + for (Pair peak : peaks) { + statistics.addValue(peak.second); + } + final double mean = statistics.getMean(); + final double stddev = statistics.getStandardDeviation(); + peaks.removeIf(p -> p.second < (mean + stddev * 2)); } - /** - * Find local maxima in the spectrum and returns the indices of those maxima as integer - * array. - * - * @param threshold Threshold for search. Values bellow that threshold won't be considered. - * @return Array containing indices (zero-based) of local maxima. - */ - public List> findLocalMaxima(double threshold, boolean significant) { - List> peaks = new ArrayList<>(); - for (int i=1;i Math.max(spectrum[i+1], spectrum[i-1])) { - peaks.add(this.get(i)); - } - } - - if (significant) { - DescriptiveStatistics statistics = new DescriptiveStatistics(); - for (Pair peak : peaks) { - statistics.addValue(peak.second); - } - final double mean = statistics.getMean(); - final double stddev = statistics.getStandardDeviation(); - peaks.removeIf(p -> p.second < (mean + stddev * 2)); - } - - return peaks; - } - - /** - * Returns the minimum frequency in the spectrum. - * - * @return Minimum frequency - */ - public float minFreq() { - return this.frequencies[0]; - } - - /** - * Returns the maximum frequency in the spectrum. - * - * @return Maximum frequency - */ - public float maxFreq() { - return this.frequencies[this.frequencies.length-1]; - } - - /** - * Returns an iterator over elements of type {@code T}. - * - * @return an Iterator. - */ - @Override - public Iterator> iterator() { - return new Iterator>() { - - private int idx = 0; - - @Override - public boolean hasNext() { - return this.idx < Spectrum.this.spectrum.length; - } - - @Override - public Pair next() { - return new Pair<>(Spectrum.this.frequencies[this.idx], Spectrum.this.spectrum[this.idx++]); - } - }; - } + return peaks; + } + + /** + * Returns the minimum frequency in the spectrum. + * + * @return Minimum frequency + */ + public float minFreq() { + return this.frequencies[0]; + } + + /** + * Returns the maximum frequency in the spectrum. + * + * @return Maximum frequency + */ + public float maxFreq() { + return this.frequencies[this.frequencies.length - 1]; + } + + /** + * Returns an iterator over elements of type {@code T}. + * + * @return an Iterator. + */ + @Override + public Iterator> iterator() { + return new Iterator>() { + + private int idx = 0; + + @Override + public boolean hasNext() { + return this.idx < Spectrum.this.spectrum.length; + } + + @Override + public Pair next() { + return new Pair<>(Spectrum.this.frequencies[this.idx], Spectrum.this.spectrum[this.idx++]); + } + }; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/BlackmanHarrisWindow.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/BlackmanHarrisWindow.java index 4f59f528e..d5dcf81c3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/BlackmanHarrisWindow.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/BlackmanHarrisWindow.java @@ -3,25 +3,27 @@ public class BlackmanHarrisWindow implements WindowFunction { - /** Constants defined for the Blackman-Harris Window. */ - private static final float a0 = 0.35875f; - private static final float a1 = 0.48829f; - private static final float a2 = 0.14128f; - private static final float a3 = 0.01168f; + /** + * Constants defined for the Blackman-Harris Window. + */ + private static final float a0 = 0.35875f; + private static final float a1 = 0.48829f; + private static final float a2 = 0.14128f; + private static final float a3 = 0.01168f; - /** - * Calculates and returns the value of the Blackman-Harris window function at position i. - * - * @param i The position for which the function value should be calculated. - * @param length Size of the window. - * @return Function value. - */ - @Override - public double value(int i, int length) { - if (i >= 0 && i <= length-1) { - return a0 - a1 * Math.cos((2 * Math.PI * i) / (length - 1)) + a2 * Math.cos((4 * Math.PI * i) / (length - 1)) - a3 * Math.cos((6 * Math.PI * i) / (length - 1)); - } else { - return 0.0; - } + /** + * Calculates and returns the value of the Blackman-Harris window function at position i. + * + * @param i The position for which the function value should be calculated. + * @param length Size of the window. + * @return Function value. + */ + @Override + public double value(int i, int length) { + if (i >= 0 && i <= length - 1) { + return a0 - a1 * Math.cos((2 * Math.PI * i) / (length - 1)) + a2 * Math.cos((4 * Math.PI * i) / (length - 1)) - a3 * Math.cos((6 * Math.PI * i) / (length - 1)); + } else { + return 0.0; } + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/HanningWindow.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/HanningWindow.java index 4a4d807ae..05cbcbe46 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/HanningWindow.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/HanningWindow.java @@ -1,42 +1,40 @@ package org.vitrivr.cineast.core.util.dsp.fft.windows; /** - * * Hanning window function as defined in [1]. Is being use for harmonic analysis with the DFT and FFT. - * - * [1] Harris, F. J. (1978). "On the use of windows for harmonic analysis with the discrete Fourier forward". - * Proceedings of the IEEE. 66 - * + *

    + * [1] Harris, F. J. (1978). "On the use of windows for harmonic analysis with the discrete Fourier forward". Proceedings of the IEEE. 66 */ public class HanningWindow implements WindowFunction { - /** - * Calculates and returns the value of the window function at position i. - * - * @param i The position for which the function value should be calculated. - * @param length Size of the window. - * @return Function value. - */ - @Override - public final double value(int i, int length) { - if (i >= 0 && i <= length-1) { - return 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (length-1)); - } else { - return 0; - } + + /** + * Calculates and returns the value of the window function at position i. + * + * @param i The position for which the function value should be calculated. + * @param length Size of the window. + * @return Function value. + */ + @Override + public final double value(int i, int length) { + if (i >= 0 && i <= length - 1) { + return 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (length - 1)); + } else { + return 0; } + } - /** - * Returns a normalization factor for the window function. - * - * @param length Length for which the normalization factor should be obtained. - * @return Normalization factor. - */ - @Override - public final double normalization(int length) { - double normal = 0.0f; - for (int i=0; i<=length; i++) { - normal += this.value(i, length); - } - return normal/length; + /** + * Returns a normalization factor for the window function. + * + * @param length Length for which the normalization factor should be obtained. + * @return Normalization factor. + */ + @Override + public final double normalization(int length) { + double normal = 0.0f; + for (int i = 0; i <= length; i++) { + normal += this.value(i, length); } + return normal / length; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/RectangularWindow.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/RectangularWindow.java index 89d1dfe51..016bdb311 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/RectangularWindow.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/RectangularWindow.java @@ -2,33 +2,33 @@ /** * IdentityWindows - this is the same as applying no window at all. - * */ public class RectangularWindow implements WindowFunction { - /** - * Returns the value of the windowing function at position i. - * - * @param i The position for which the function value should be calculated. - * @param length Size of the window. - * @return Function value. - */ - @Override - public final double value(int i, int length) { - if (i >= 0 && i <= length-1) { - return 1.0; - } else { - return 0.0; - } - } - /** - * Returns a normalization factor for the window function. - * - * @param length Length for which the normalization factor should be obtained. - * @return Normalization factor. - */ - @Override - public final double normalization(int length) { - return 1.0; + /** + * Returns the value of the windowing function at position i. + * + * @param i The position for which the function value should be calculated. + * @param length Size of the window. + * @return Function value. + */ + @Override + public final double value(int i, int length) { + if (i >= 0 && i <= length - 1) { + return 1.0; + } else { + return 0.0; } + } + + /** + * Returns a normalization factor for the window function. + * + * @param length Length for which the normalization factor should be obtained. + * @return Normalization factor. + */ + @Override + public final double normalization(int length) { + return 1.0; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/WindowFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/WindowFunction.java index 8ea7075a6..7dc849b5d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/WindowFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/fft/windows/WindowFunction.java @@ -2,44 +2,44 @@ /** * Interface implemented by any windows function (e.g. used for FFT). - * */ public interface WindowFunction { - /** - * Calculates and returns the value of the window function at position i. - * - * @param i The position for which the function value should be calculated. - * @param length Size of the window. - * @return Function value. - */ - double value(int i, int length); + /** + * Calculates and returns the value of the window function at position i. + * + * @param i The position for which the function value should be calculated. + * @param length Size of the window. + * @return Function value. + */ + double value(int i, int length); - /** - * Calculates and returns a normalization factor for the window function. - * - * @param length Length for which the normalization factor should be obtained. - * @return Normalization factor. - */ - default double normalization(int length) { - double normal = 0.0f; - for (int i=0; i<=length; i++) { - normal += this.value(i, length); - } - return normal/length; + + /** + * Calculates and returns a normalization factor for the window function. + * + * @param length Length for which the normalization factor should be obtained. + * @return Normalization factor. + */ + default double normalization(int length) { + double normal = 0.0f; + for (int i = 0; i <= length; i++) { + normal += this.value(i, length); } + return normal / length; + } - /** - * Calculates and returns the values of a WindowFunction. - * - * @param length Size of the window. - * @return Array containing the values. - */ - default double[] values(int length) { - double[] window = new double[length]; - for (int i = 0; i + * [1] Klapuri, A. (2006). Multiple Fundamental Frequency Estimation by Summing Harmonic Amplitudes. Proceedings of the International Symposium/Conference on Music Information Retrieval (ISMIR), 216–221. */ public class SpectralWhiteningFilter implements FrequencyDomainFilterInterface { - /** - * Represents a filter-band Hb with a triangular response. - */ - public class FilterBand { - /** Index of the lowest band-frequency (cut-off) in the provided FFT bin array. */ - private final int lowerBandIndex; - - /** Center-frequency of the band. */ - private final int centerBandIndex; - - /** Index of the highest band-frequency (cut-off) in the provided FFT bin array. */ - private final int upperBandIndex; - - /** Triangular weights in the filter band.*/ - private float[] weights; - - /** - * Constructor for filter band. - * - * @param bandindex Index of the filter band. - */ - private FilterBand(int bandindex) { - this.lowerBandIndex = FFTUtil.binIndex(SpectralWhiteningFilter.this.centerfrequencies[bandindex-1], SpectralWhiteningFilter.this.windowsize, SpectralWhiteningFilter.this.samplingrate); - this.centerBandIndex = FFTUtil.binIndex(SpectralWhiteningFilter.this.centerfrequencies[bandindex], SpectralWhiteningFilter.this.windowsize, SpectralWhiteningFilter.this.samplingrate); - this.upperBandIndex = FFTUtil.binIndex(SpectralWhiteningFilter.this.centerfrequencies[bandindex+1], SpectralWhiteningFilter.this.windowsize, SpectralWhiteningFilter.this.samplingrate); - - int length = upperBandIndex-lowerBandIndex+1; - - this.weights = new float[length]; - for (int i=0;iImportant: This method is applied in place and changes the values - * in the original array. - * - * @param fftbins Array of complex FFT bins. - * @return FFT bins after application of spectral whitening. + * @param bandindex Index of the filter band. */ - @Override - public Complex[] filterInPlace(Complex[] fftbins) { - /* Calculates the compression factor per band. */ - double[] compression = new double[filterbands.length]; - for (int i=0; iImportant: This method is applied in place and changes the values + * in the original array. + * + * @param fftbins Array of complex FFT bins. + * @return FFT bins after application of spectral whitening. + */ + @Override + public Complex[] filterInPlace(Complex[] fftbins) { + /* Calculates the compression factor per band. */ + double[] compression = new double[filterbands.length]; + for (int i = 0; i < filterbands.length; i++) { + FilterBand band = this.filterbands[i]; + double sigma = 0.0; + for (int j = band.lowerBandIndex; j <= band.upperBandIndex; j++) { + sigma += band.weights[j - band.lowerBandIndex] * Math.pow(fftbins[j].abs(), 2); + } + compression[i] = Math.pow(Math.sqrt(sigma / fftbins.length), this.amount - 1); + } + + /* Applies the linear interpolation of two compression factors between two center frequencies. */ + for (int i = 0; i < filterbands.length - 1; i++) { + FilterBand band0 = this.filterbands[i]; + FilterBand band1 = this.filterbands[i + 1]; + + double c0 = compression[i]; + double c1 = compression[i + 1]; + double scale = (c1 - c0) / (band1.centerBandIndex - band0.centerBandIndex); + + for (int k = band0.centerBandIndex; k < band1.centerBandIndex; k++) { + fftbins[k] = fftbins[k].multiply(c0 + (k - band0.centerBandIndex) * scale); + } } + return fftbins; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/filter/spectrum/SpectralWhiteningFilter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/filter/spectrum/SpectralWhiteningFilter.java index 4b3ece97b..59641c85e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/filter/spectrum/SpectralWhiteningFilter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/dsp/filter/spectrum/SpectralWhiteningFilter.java @@ -6,91 +6,93 @@ /** * Implements a spectral whitening filter as described in [1]. - * - * [1] Klapuri, A. P. (2003). Multiple Fundamental Frequency Estimation Based on Harmonicity and Spectral Smoothness. - * IEEE Transactions on Speech and Audio Processing, 11(6), 804–816. http://doi.org/10.1109/TSA.2003.815516 - * + *

    + * [1] Klapuri, A. P. (2003). Multiple Fundamental Frequency Estimation Based on Harmonicity and Spectral Smoothness. IEEE Transactions on Speech and Audio Processing, 11(6), 804–816. http://doi.org/10.1109/TSA.2003.815516 */ -public class SpectralWhiteningFilter implements SpectrumFilterInterface{ +public class SpectralWhiteningFilter implements SpectrumFilterInterface { - /** The maximum frequency to consider for spectral whitening. Frequencies above that threshold will be left untouched. */ - private final float maxFrequency; + /** + * The maximum frequency to consider for spectral whitening. Frequencies above that threshold will be left untouched. + */ + private final float maxFrequency; - /** The minimum frequency to consider for spectral whitening. Frequencies bellow that threshold will be left untouched. */ - private final float minFrequency; + /** + * The minimum frequency to consider for spectral whitening. Frequencies bellow that threshold will be left untouched. + */ + private final float minFrequency; - /** - * Constructor for SpectralWhiteningFilter - * - * @param minFrequency The minimum frequency to consider for spectral whitening. - * @param maxFrequency The maximum frequency to consider for spectral whitening. - */ - public SpectralWhiteningFilter(float minFrequency, float maxFrequency) { - this.minFrequency = minFrequency; - this.maxFrequency = maxFrequency; - - } + /** + * Constructor for SpectralWhiteningFilter + * + * @param minFrequency The minimum frequency to consider for spectral whitening. + * @param maxFrequency The maximum frequency to consider for spectral whitening. + */ + public SpectralWhiteningFilter(float minFrequency, float maxFrequency) { + this.minFrequency = minFrequency; + this.maxFrequency = maxFrequency; + } - /** Returns a filtered power spectrum by first warping the magnitude of the original power spectrum - * and then subtracting the moving average. - * - * @param fft FFT to derive the spectrum from. - * @return Spectral whitened power spectrum - */ - @Override - public Spectrum filteredSpectrum(FFT fft) { - final int size = fft.getWindowsize()/2; - final int min_index = FFTUtil.binIndex(this.minFrequency, fft.getWindowsize(), fft.getSamplingrate()); - final int max_index = FFTUtil.binIndex(this.maxFrequency, fft.getWindowsize(), fft.getSamplingrate()); - final double g = this.scalingFactor(fft, min_index, max_index); - /* Prepare array holding the whitened spectrum. */ - double[] whitened = new double[size]; + /** + * Returns a filtered power spectrum by first warping the magnitude of the original power spectrum and then subtracting the moving average. + * + * @param fft FFT to derive the spectrum from. + * @return Spectral whitened power spectrum + */ + @Override + public Spectrum filteredSpectrum(FFT fft) { + final int size = fft.getWindowsize() / 2; + final int min_index = FFTUtil.binIndex(this.minFrequency, fft.getWindowsize(), fft.getSamplingrate()); + final int max_index = FFTUtil.binIndex(this.maxFrequency, fft.getWindowsize(), fft.getSamplingrate()); + final double g = this.scalingFactor(fft, min_index, max_index); - /* Performs magnitude wrapping. */ - for (int n=0;n spectra, int width, int height) { - - if (spectra.isEmpty()) { - return null; - } - - BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + /** + * YlOrRd palette used for visualization. + */ + private final static Color[] YlOrRd = { + new Color(255, 255, 255), + new Color(255, 255, 234), + new Color(255, 237, 160), + new Color(254, 217, 118), + new Color(254, 178, 76), + new Color(253, 141, 61), + new Color(252, 78, 42), + new Color(227, 26, 28), + new Color(189, 0, 38), + new Color(128, 0, 38), + new Color(100, 0, 32), + new Color(80, 0, 16), + new Color(0, 0, 0), + }; + + private AudioSignalVisualizer() { + } + + + /** + * This method visualizes a spectrogram that shows temporal development of a signals spectogram (i.e time vs. frequency vs power distribution). + *

    + * x-axis: Relative time y-axis: Frequency bins color: Power in bin (dB) + * + * @param spectra List of spectra (one per time-slot) + * @param width Width of spectrogram in pixels. + * @param height Height of spectrogram in pixels. + * @return BufferedImage containing the spectogram. + */ + public static BufferedImage visualizeSpectogram(List spectra, int width, int height) { + + if (spectra.isEmpty()) { + return null; + } + BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); - final float width_time_ratio = (float)spectra.size()/((float)width); - final float height_freq_ratio = (float)spectra.get(0).size()/((float)height); + final float width_time_ratio = (float) spectra.size() / ((float) width); + final float height_freq_ratio = (float) spectra.get(0).size() / ((float) height); - for (int x=0;x max) { - throw new IllegalArgumentException("Minimum must be smaller than maximum."); - } - if (Double.isNaN(value) || value == 0.0f) { - return YlOrRd[YlOrRd.length-1]; - } - double ratio = (YlOrRd.length-1)/(max-min); - int idx = (YlOrRd.length-1) - (int)((value-min) * ratio); + return image; + } + + /** + * Returns the color-code for a dB value given a range that should be color-coded. + * + * @param min Minimal value in dB that should be color-coded. + * @param max Maximal value in dB should be color-coded. + * @param value value for which a color-code is required. + */ + public static Color color(double min, double max, double value) { + if (min > max) { + throw new IllegalArgumentException("Minimum must be smaller than maximum."); + } + if (Double.isNaN(value) || value == 0.0f) { + return YlOrRd[YlOrRd.length - 1]; + } + double ratio = (YlOrRd.length - 1) / (max - min); + int idx = (YlOrRd.length - 1) - (int) ((value - min) * ratio); - if (idx > YlOrRd.length-1) { - return YlOrRd[YlOrRd.length-1]; - } - if (idx < 0) { - return YlOrRd[0]; - } - return YlOrRd[idx]; + if (idx > YlOrRd.length - 1) { + return YlOrRd[YlOrRd.length - 1]; + } + if (idx < 0) { + return YlOrRd[0]; } + return YlOrRd[idx]; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ContourHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ContourHelper.java index e1f4ea4dc..cc7c642a0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ContourHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ContourHelper.java @@ -13,234 +13,231 @@ import boofcv.struct.image.Planar; import georegression.metric.UtilAngle; import georegression.struct.point.Point2D_I32; - -import java.awt.*; +import java.awt.Color; +import java.awt.Graphics; import java.awt.image.BufferedImage; import java.util.List; public final class ContourHelper { - /** - * Private constructor; do not instantiate! - */ - private ContourHelper() {} - - /** - * Applies a contour-detection algorithm on the provided image and returns a list of detected contours. First, the image - * is converted to a BinaryImage using a threshold algorithm (Otsu). Afterwards, blobs in the image are detected using - * an 8-connect rule. - * - * This method provides the best results if the image is a black & white, i.e. factually binary, image! - * See {@link ContourHelper#segmentImageByColour(BufferedImage,float[])} to convert a coloured image to a binary image. - * - * @param image BufferedImage in which contours should be detected. - * @return List of contours. - */ - public static List getContours(BufferedImage image) { - /* Draw a black frame around to image so as to make sure that all detected contours are internal contours. */ - BufferedImage resized = new BufferedImage(image.getWidth() + 4, image.getHeight() + 4, image.getType()); - Graphics g = resized.getGraphics(); - g.setColor(Color.BLACK); - g.fillRect(0,0,resized.getWidth(),resized.getHeight()); - g.drawImage(image, 2,2, image.getWidth(), image.getHeight(), null); - - /* Convert to BufferedImage to Gray-scale image and prepare Binary image. */ - GrayF32 input = ConvertBufferedImage.convertFromSingle(resized, null, GrayF32.class); - GrayU8 binary = new GrayU8(input.width,input.height); - GrayS32 label = new GrayS32(input.width,input.height); - - /* Select a global threshold using Otsu's method and apply that threshold. */ - double threshold = GThresholdImageOps.computeOtsu(input, 0, 255); - ThresholdImageOps.threshold(input, binary,(float)threshold,true); - - /* Remove small blobs through erosion and dilation; The null in the input indicates that it should internally - * declare the work image it needs this is less efficient, but easier to code. */ - GrayU8 filtered = BinaryImageOps.erode8(binary, 1, null); - filtered = BinaryImageOps.dilate8(filtered, 1, null); - - /* Detect blobs inside the image using an 8-connect rule. */ - return BinaryImageOps.contour(filtered, ConnectRule.EIGHT, label); + /** + * Private constructor; do not instantiate! + */ + private ContourHelper() { + } + + /** + * Applies a contour-detection algorithm on the provided image and returns a list of detected contours. First, the image is converted to a BinaryImage using a threshold algorithm (Otsu). Afterwards, blobs in the image are detected using an 8-connect rule. + *

    + * This method provides the best results if the image is a black & white, i.e. factually binary, image! See {@link ContourHelper#segmentImageByColour(BufferedImage, float[])} to convert a coloured image to a binary image. + * + * @param image BufferedImage in which contours should be detected. + * @return List of contours. + */ + public static List getContours(BufferedImage image) { + /* Draw a black frame around to image so as to make sure that all detected contours are internal contours. */ + BufferedImage resized = new BufferedImage(image.getWidth() + 4, image.getHeight() + 4, image.getType()); + Graphics g = resized.getGraphics(); + g.setColor(Color.BLACK); + g.fillRect(0, 0, resized.getWidth(), resized.getHeight()); + g.drawImage(image, 2, 2, image.getWidth(), image.getHeight(), null); + + /* Convert to BufferedImage to Gray-scale image and prepare Binary image. */ + GrayF32 input = ConvertBufferedImage.convertFromSingle(resized, null, GrayF32.class); + GrayU8 binary = new GrayU8(input.width, input.height); + GrayS32 label = new GrayS32(input.width, input.height); + + /* Select a global threshold using Otsu's method and apply that threshold. */ + double threshold = GThresholdImageOps.computeOtsu(input, 0, 255); + ThresholdImageOps.threshold(input, binary, (float) threshold, true); + + /* Remove small blobs through erosion and dilation; The null in the input indicates that it should internally + * declare the work image it needs this is less efficient, but easier to code. */ + GrayU8 filtered = BinaryImageOps.erode8(binary, 1, null); + filtered = BinaryImageOps.dilate8(filtered, 1, null); + + /* Detect blobs inside the image using an 8-connect rule. */ + return BinaryImageOps.contour(filtered, ConnectRule.EIGHT, label); + } + + + /** + * Segments a colored image by identifying the average color in a given square region and turning all pixels that are close to that color to white. + * + * @param image The image that should be converted + * @param startX The x-coordinate of the top-left corner of the square region. + * @param startY The y-coordinate of the top-left corner of the square region. + * @param size The size of the square region. + * @return Converted image where pixels close to the provided color are white and the others are black + */ + public static BufferedImage segmentImageByColor(BufferedImage image, int startX, int startY, int size) { + final float[] avgColor = new float[]{0.0f, 0.0f, 0.0f}; + for (int x = startX; x < startX + size; x++) { + for (int y = startY; y < startY + size; y++) { + int rgb = image.getRGB(x, y); + avgColor[0] += (rgb >> 16) & 0xFF; + avgColor[1] += (rgb >> 8) & 0xFF; + avgColor[2] += rgb & 0xFF; + } } - - - /** - * Segments a colored image by identifying the average color in a given square region and turning all pixels that are - * close to that color to white. - * - * @param image The image that should be converted - * @param startX The x-coordinate of the top-left corner of the square region. - * @param startY The y-coordinate of the top-left corner of the square region. - * @param size The size of the square region. - * @return Converted image where pixels close to the provided color are white and the others are black - */ - public static BufferedImage segmentImageByColor(BufferedImage image, int startX, int startY, int size) { - final float[] avgColor = new float[]{0.0f,0.0f,0.0f}; - for (int x = startX; x> 16) & 0xFF; - avgColor[1] += (rgb >> 8) & 0xFF; - avgColor[2] += rgb & 0xFF; - } + avgColor[0] /= size * size; + avgColor[1] /= size * size; + avgColor[2] /= size * size; + return segmentImageByColour(image, avgColor); + } + + /** + * Segments a colored image by turning all pixels that are close to the provided color to white. + * + * @param image The image that should be converted. + * @param colorRgb The colour that should be turned to white. + * @return Converted image where pixels close to the provided color are white and the others are black + */ + public static BufferedImage segmentImageByColour(BufferedImage image, float[] colorRgb) { + /* Phase 1): Convert average RGB color to HSV. */ + final float[] avgHsvColor = new float[]{0.0f, 0.0f, 0.0f}; + ColorHsv.rgbToHsv(colorRgb[0], colorRgb[1], colorRgb[2], avgHsvColor); + + /* Phase 2a): Convert the input BufferedImage to a HSV image and extract hue and saturation bands, which are independent of intensity. */ + final Planar input = ConvertBufferedImage.convertFromPlanar(image, null, true, GrayF32.class); + final Planar hsv = input.createSameShape(); + ColorHsv.rgbToHsv_F32(input, hsv); + + final GrayF32 H = hsv.getBand(0); + final GrayF32 S = hsv.getBand(1); + + /* Phase 2b): Determine thresholds. */ + float maxDist2 = 0.4f * 0.4f; + float adjustUnits = (float) (Math.PI / 2.0); + + /* Phase 3): For each pixel in the image, determine distance to average color. If color is closed, turn pixel white. */ + final BufferedImage output = new BufferedImage(input.width, input.height, BufferedImage.TYPE_INT_RGB); + for (int y = 0; y < hsv.height; y++) { + for (int x = 0; x < hsv.width; x++) { + // Hue is an angle in radians, so simple subtraction doesn't work + float dh = UtilAngle.dist(H.unsafe_get(x, y), avgHsvColor[0]); + float ds = (S.unsafe_get(x, y) - avgHsvColor[1]) * adjustUnits; + + // this distance measure is a bit naive, but good enough for to demonstrate the concept + float dist2 = dh * dh + ds * ds; + if (dist2 <= maxDist2) { + output.setRGB(x, y, Color.WHITE.getRGB()); } - avgColor[0] /= size*size; - avgColor[1] /= size*size; - avgColor[2] /= size*size; - return segmentImageByColour(image, avgColor); + } } - - /** - * Segments a colored image by turning all pixels that are close to the provided color to white. - * - * @param image The image that should be converted. - * @param colorRgb The colour that should be turned to white. - * @return Converted image where pixels close to the provided color are white and the others are black - */ - public static BufferedImage segmentImageByColour(BufferedImage image, float[] colorRgb) { - /* Phase 1): Convert average RGB color to HSV. */ - final float[] avgHsvColor = new float[]{0.0f,0.0f,0.0f}; - ColorHsv.rgbToHsv(colorRgb[0], colorRgb[1], colorRgb[2], avgHsvColor); - - /* Phase 2a): Convert the input BufferedImage to a HSV image and extract hue and saturation bands, which are independent of intensity. */ - final Planar input = ConvertBufferedImage.convertFromPlanar(image,null, true, GrayF32.class); - final Planar hsv = input.createSameShape(); - ColorHsv.rgbToHsv_F32(input,hsv); - - final GrayF32 H = hsv.getBand(0); - final GrayF32 S = hsv.getBand(1); - - /* Phase 2b): Determine thresholds. */ - float maxDist2 = 0.4f*0.4f; - float adjustUnits = (float)(Math.PI/2.0); - - /* Phase 3): For each pixel in the image, determine distance to average color. If color is closed, turn pixel white. */ - final BufferedImage output = new BufferedImage(input.width,input.height,BufferedImage.TYPE_INT_RGB); - for(int y = 0; y < hsv.height; y++) { - for(int x = 0; x < hsv.width; x++) { - // Hue is an angle in radians, so simple subtraction doesn't work - float dh = UtilAngle.dist(H.unsafe_get(x,y),avgHsvColor[0]); - float ds = (S.unsafe_get(x,y)-avgHsvColor[1])*adjustUnits; - - // this distance measure is a bit naive, but good enough for to demonstrate the concept - float dist2 = dh*dh + ds*ds; - if( dist2 <= maxDist2 ) { - output.setRGB(x,y,Color.WHITE.getRGB()); - } - } - } - return output; + return output; + } + + + /** + * Calculates and returns the position of the centroid given a contour. + * + * @param contour List of points that make up the contour. + * @return Coordinates of the centroid. + */ + public static Point2D_I32 centroid(List contour) { + Point2D_I32 centroid = new Point2D_I32(); + + for (Point2D_I32 point : contour) { + centroid.x += point.x; + centroid.y += point.y; } - - /** - * Calculates and returns the position of the centroid given a contour. - * - * @param contour List of points that make up the contour. - * @return Coordinates of the centroid. - */ - public static Point2D_I32 centroid(List contour) { - Point2D_I32 centroid = new Point2D_I32(); - - for (Point2D_I32 point : contour) { - centroid.x += point.x; - centroid.y += point.y; - } - - centroid.x /= contour.size(); - centroid.y /= contour.size(); - return centroid; + centroid.x /= contour.size(); + centroid.y /= contour.size(); + return centroid; + } + + /** + * Determines whether or not a given image is a binary image. This method first checks the type of the image. If by type, the image is non-binary, the individual pixels are checked. + * + * @param image The image that should be checked. + * @return True if image is a binary image. False otherwise. + */ + public static boolean isBinary(BufferedImage image) { + if (image.getType() == BufferedImage.TYPE_BYTE_BINARY) { + return true; } - - /** - * Determines whether or not a given image is a binary image. This method first checks the type of the image. If by - * type, the image is non-binary, the individual pixels are checked. - * - * @param image The image that should be checked. - * @return True if image is a binary image. False otherwise. - */ - public static boolean isBinary(BufferedImage image) { - if (image.getType() == BufferedImage.TYPE_BYTE_BINARY) return true; - final int black = Color.black.getRGB(); - final int white = Color.white.getRGB(); - for (int x = 0; x contour, boolean pad) { - Point2D_I32 centroid = centroid(contour); - - int size = 0; - if (pad) { - double base2log = Math.log(contour.size())/Math.log(2); - if (Math.round(base2log) >= base2log) { - size = (int) Math.pow(2,(Math.round(base2log))); - } else { - size = (int) Math.pow(2,(Math.round(base2log) + 1)); - } - } else { - size = contour.size(); - } - - double[] distance = new double[size]; - for (int i = 0;i contour, boolean pad) { + Point2D_I32 centroid = centroid(contour); + + int size = 0; + if (pad) { + double base2log = Math.log(contour.size()) / Math.log(2); + if (Math.round(base2log) >= base2log) { + size = (int) Math.pow(2, (Math.round(base2log))); + } else { + size = (int) Math.pow(2, (Math.round(base2log) + 1)); + } + } else { + size = contour.size(); } - /** - * Calculates and returns the bounds for the provided 2D shape. - * - * @param vertices List of vertices that make up the shape for which bounds should be calculated. - * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y} - */ - public static int[] bounds(List vertices) { - /* If no vertices are in the list, the box is zero. */ - if (vertices.isEmpty()) { - return new int[4]; - } + double[] distance = new double[size]; + for (int i = 0; i < size; i++) { + if (i < contour.size()) { + Point2D_I32 point = contour.get(i); + distance[i] = Math.sqrt(Math.pow(point.x - centroid.x, 2) + Math.pow(point.y - centroid.y, 2)); + } else { + distance[i] = 0; + } + } - /* Initialize the bounding-box. */ - int[] bounds = {Integer.MAX_VALUE, -Integer.MAX_VALUE, Integer.MAX_VALUE, -Integer.MAX_VALUE}; - - /* Find max and min y-values. */ - for(Point2D_I32 vertex : vertices) { - if (vertex.x < bounds[0]) { - bounds[0] = vertex.x; - } - if (vertex.x > bounds[1]) { - bounds[1] = vertex.x; - } - if (vertex.y < bounds[2]) { - bounds[2] = vertex.y; - } - if (vertex.y > bounds[3]) { - bounds[3] = vertex.y; - } - } + return distance; + } + + /** + * Calculates and returns the bounds for the provided 2D shape. + * + * @param vertices List of vertices that make up the shape for which bounds should be calculated. + * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y} + */ + public static int[] bounds(List vertices) { + /* If no vertices are in the list, the box is zero. */ + if (vertices.isEmpty()) { + return new int[4]; + } - /* Return bounding-box. */ - return bounds; + /* Initialize the bounding-box. */ + int[] bounds = {Integer.MAX_VALUE, -Integer.MAX_VALUE, Integer.MAX_VALUE, -Integer.MAX_VALUE}; + + /* Find max and min y-values. */ + for (Point2D_I32 vertex : vertices) { + if (vertex.x < bounds[0]) { + bounds[0] = vertex.x; + } + if (vertex.x > bounds[1]) { + bounds[1] = vertex.x; + } + if (vertex.y < bounds[2]) { + bounds[2] = vertex.y; + } + if (vertex.y > bounds[3]) { + bounds[3] = vertex.y; + } } + + /* Return bounding-box. */ + return bounds; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/HOGHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/HOGHelper.java index 70af1c55d..8a581a17f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/HOGHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/HOGHelper.java @@ -7,73 +7,69 @@ import boofcv.struct.feature.TupleDesc_F64; import boofcv.struct.image.GrayU8; import boofcv.struct.image.ImageType; - import java.awt.image.BufferedImage; /** - * This class provides methods to extract Histogram of Oriented Gradients (HOG) features from images using - * the BoofCV library. The default settings for the HOG features are taken from [1]. - * - * [1] Velmurugan, K., & Santosh Baboo, S. (2011). - * Image Retrieval using Harris Corners and Histogram of Oriented Gradients. International Journal of Computer Applications, 24(7), 6–10. Retrieved from http://www.ijcaonline.org/journal/number14/pxc387478.pdf - * + * This class provides methods to extract Histogram of Oriented Gradients (HOG) features from images using the BoofCV library. The default settings for the HOG features are taken from [1]. + *

    + * [1] Velmurugan, K., & Santosh Baboo, S. (2011). Image Retrieval using Harris Corners and Histogram of Oriented Gradients. International Journal of Computer Applications, 24(7), 6–10. Retrieved from http://www.ijcaonline.org/journal/number14/pxc387478.pdf */ public final class HOGHelper { - /** Default configuration for HOG descriptors according to [1] - * - * - Number of cells: 4 in each dimension - * - Number of bins: 8 - * - Number of pixels per cell: 16 x 16 - */ - public static final ConfigDenseHoG DEFAULT_CONFIG = new ConfigDenseHoG(); - static { - DEFAULT_CONFIG.pixelsPerCell = 16; - DEFAULT_CONFIG.cellsPerBlockX = 4; - DEFAULT_CONFIG.cellsPerBlockY = 4; - DEFAULT_CONFIG.orientationBins = 8; - DEFAULT_CONFIG.fastVariant = false; - } + /** + * Default configuration for HOG descriptors according to [1] + *

    + * - Number of cells: 4 in each dimension - Number of bins: 8 - Number of pixels per cell: 16 x 16 + */ + public static final ConfigDenseHoG DEFAULT_CONFIG = new ConfigDenseHoG(); + + static { + DEFAULT_CONFIG.pixelsPerCell = 16; + DEFAULT_CONFIG.cellsPerBlockX = 4; + DEFAULT_CONFIG.cellsPerBlockY = 4; + DEFAULT_CONFIG.orientationBins = 8; + DEFAULT_CONFIG.fastVariant = false; + } - /** - * Private constructor; do not instantiate! - */ - private HOGHelper() { + /** + * Private constructor; do not instantiate! + */ + private HOGHelper() { - } + } - /** - * Returns HOG descriptors for an image using the following default configuration. - * - * @param image Image for which to obtain the HOG descriptors. - * @return DescribeImageDense object containing the HOG descriptor. - */ - public static DescribeImageDense getHOGDescriptors(BufferedImage image) { - return getHOGDescriptors(image, DEFAULT_CONFIG); - } + /** + * Returns HOG descriptors for an image using the following default configuration. + * + * @param image Image for which to obtain the HOG descriptors. + * @return DescribeImageDense object containing the HOG descriptor. + */ + public static DescribeImageDense getHOGDescriptors(BufferedImage image) { + return getHOGDescriptors(image, DEFAULT_CONFIG); + } - /** - * Returns HOG descriptors for an image using the provided settings. - * - * @param image Image for which to obtain the HOG descriptors. - * @param config ConfigDenseHog object that specifies the parameters for the HOG algorithm. - * @return DescribeImageDense object containing the HOG descriptor. - */ - public static DescribeImageDense getHOGDescriptors(BufferedImage image, ConfigDenseHoG config) { - GrayU8 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayU8.class); - DescribeImageDense desc = FactoryDescribeImageDense.hog(config, ImageType.single(GrayU8.class)); - desc.process(gray); - return desc; - } + /** + * Returns HOG descriptors for an image using the provided settings. + * + * @param image Image for which to obtain the HOG descriptors. + * @param config ConfigDenseHog object that specifies the parameters for the HOG algorithm. + * @return DescribeImageDense object containing the HOG descriptor. + */ + public static DescribeImageDense getHOGDescriptors(BufferedImage image, ConfigDenseHoG config) { + GrayU8 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayU8.class); + DescribeImageDense desc = FactoryDescribeImageDense.hog(config, ImageType.single(GrayU8.class)); + desc.process(gray); + return desc; + } - /** - * Returns the size of a HOG vector given a configuration. - * - * @param config ConfigDenseHoG that describes the HOG configuration. - * @return Size of an individual HOG feature. - */ - public static int hogVectorSize (ConfigDenseHoG config) { - return config.cellsPerBlockX * config.cellsPerBlockX * config.orientationBins; - } + /** + * Returns the size of a HOG vector given a configuration. + * + * @param config ConfigDenseHoG that describes the HOG configuration. + * @return Size of an individual HOG feature. + */ + public static int hogVectorSize(ConfigDenseHoG config) { + return config.cellsPerBlockX * config.cellsPerBlockX * config.orientationBins; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java index 5a96a7712..492b8279b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/SURFHelper.java @@ -6,68 +6,77 @@ import boofcv.io.image.ConvertBufferedImage; import boofcv.struct.feature.BrightFeature; import boofcv.struct.image.GrayF32; - import java.awt.image.BufferedImage; /** * Extracts images based on the SURF algorithm described in [1]. For feature detection, the Fast Hession (FH-9) algorithm is used. - * + *

    * [1] Herbert Bay, Andreas Ess, Tinne Tuytelaars, and Luc Van Gool, "Speeded-Up Robust Features (SURF)", CVIU June, 2008, Volume 110, Issue 3, pages 346-359 - * */ public final class SURFHelper { - /** Number of Octaves to consider in the interest point detection step (Fast Hessian) of SURF. Defaults to 4 for FH-9 described in [1]. */ - private final static int FH_NUMBER_OF_OCTAVES = 4; + /** + * Number of Octaves to consider in the interest point detection step (Fast Hessian) of SURF. Defaults to 4 for FH-9 described in [1]. + */ + private final static int FH_NUMBER_OF_OCTAVES = 4; - /** Number of scales to consider in the interest point detection step (Fast Hessian) of SURF. Defaults to 4 for FH-9 described in [1]. */ - private final static int FH_NUMBER_SCALES_PER_OCTAVE = 4; + /** + * Number of scales to consider in the interest point detection step (Fast Hessian) of SURF. Defaults to 4 for FH-9 described in [1]. + */ + private final static int FH_NUMBER_SCALES_PER_OCTAVE = 4; - /** How often pixels are sampled in the first octave during the interest point detection step (Fast Hessian) of SURF. Defaults to 1 for FH-9 described in [1] */ - private final static int FH_INITIAL_SAMPLE_SIZE = 1; + /** + * How often pixels are sampled in the first octave during the interest point detection step (Fast Hessian) of SURF. Defaults to 1 for FH-9 described in [1] + */ + private final static int FH_INITIAL_SAMPLE_SIZE = 1; - /** Size/width of the smallest feature/kernel in the first octave during the interest point detection step (Fast Hessian) f SURF. Defaults to 9 for FH-9 described in [1] */ - private final static int FH_INITIAL_SIZE = 9; + /** + * Size/width of the smallest feature/kernel in the first octave during the interest point detection step (Fast Hessian) f SURF. Defaults to 9 for FH-9 described in [1] + */ + private final static int FH_INITIAL_SIZE = 9; - /** Limits the number of images obtained per scale and thus the number images persisted during the extraction. */ - private final static int FH_MAX_FEATURES_PER_SCALE = -1; + /** + * Limits the number of images obtained per scale and thus the number images persisted during the extraction. + */ + private final static int FH_MAX_FEATURES_PER_SCALE = -1; - /** Size of the SURF descriptor. */ - public final static int SURF_VECTOR_SIZE = 64; + /** + * Size of the SURF descriptor. + */ + public final static int SURF_VECTOR_SIZE = 64; - /** - * Private constructor; do not instantiate! - */ - private SURFHelper() { + /** + * Private constructor; do not instantiate! + */ + private SURFHelper() { - } + } - /** - * Returns SURF descriptors for an image using the settings above. Uses the BoofCV stable SURF algorithm. - * - * @param image Image for which to obtain the SURF descriptors. - */ - public static DetectDescribePoint getStableSurf(BufferedImage image) { - /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ - GrayF32 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); - ConfigFastHessian config = new ConfigFastHessian(0, 2, FH_MAX_FEATURES_PER_SCALE, FH_INITIAL_SAMPLE_SIZE, FH_INITIAL_SIZE, FH_NUMBER_SCALES_PER_OCTAVE, FH_NUMBER_OF_OCTAVES); - DetectDescribePoint surf = FactoryDetectDescribe.surfStable(config, null, null, GrayF32.class); - surf.detect(gray); - return surf; - } + /** + * Returns SURF descriptors for an image using the settings above. Uses the BoofCV stable SURF algorithm. + * + * @param image Image for which to obtain the SURF descriptors. + */ + public static DetectDescribePoint getStableSurf(BufferedImage image) { + /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ + GrayF32 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); + ConfigFastHessian config = new ConfigFastHessian(0, 2, FH_MAX_FEATURES_PER_SCALE, FH_INITIAL_SAMPLE_SIZE, FH_INITIAL_SIZE, FH_NUMBER_SCALES_PER_OCTAVE, FH_NUMBER_OF_OCTAVES); + DetectDescribePoint surf = FactoryDetectDescribe.surfStable(config, null, null, GrayF32.class); + surf.detect(gray); + return surf; + } - /** - * Returns SURF descriptors for an image using the settings above. Uses the BoofCV fast SURF algorithm, - * which yields less images but operates a bit faster. - * - * @param image Image for which to obtain the SURF descriptors. - */ - public static DetectDescribePoint getFastSurf(BufferedImage image) { - /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ - GrayF32 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); - ConfigFastHessian config = new ConfigFastHessian(0, 2, FH_MAX_FEATURES_PER_SCALE, FH_INITIAL_SAMPLE_SIZE, FH_INITIAL_SIZE, FH_NUMBER_SCALES_PER_OCTAVE, FH_NUMBER_OF_OCTAVES); - DetectDescribePoint surf = FactoryDetectDescribe.surfFast(config, null, null, GrayF32.class); - surf.detect(gray); - return surf; - } + /** + * Returns SURF descriptors for an image using the settings above. Uses the BoofCV fast SURF algorithm, which yields less images but operates a bit faster. + * + * @param image Image for which to obtain the SURF descriptors. + */ + public static DetectDescribePoint getFastSurf(BufferedImage image) { + /* Obtain raw SURF descriptors using the configuration above (FH-9 according to [1]). */ + GrayF32 gray = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); + ConfigFastHessian config = new ConfigFastHessian(0, 2, FH_MAX_FEATURES_PER_SCALE, FH_INITIAL_SAMPLE_SIZE, FH_INITIAL_SIZE, FH_NUMBER_SCALES_PER_OCTAVE, FH_NUMBER_OF_OCTAVES); + DetectDescribePoint surf = FactoryDetectDescribe.surfFast(config, null, null, GrayF32.class); + surf.detect(gray); + return surf; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java index f368fd0b5..01eb42ecf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/images/ZernikeHelper.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.util.images; +import static java.awt.image.BufferedImage.TYPE_INT_RGB; + import boofcv.alg.filter.binary.BinaryImageOps; import boofcv.alg.filter.binary.Contour; import boofcv.alg.filter.binary.GThresholdImageOps; @@ -10,168 +12,158 @@ import boofcv.struct.image.GrayS32; import boofcv.struct.image.GrayU8; import georegression.struct.point.Point2D_I32; -import org.apache.commons.math3.complex.Complex; -import org.vitrivr.cineast.core.util.math.ZernikeMoments; -import org.vitrivr.cineast.core.util.math.functions.ZernikeBasisFunction; - import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.List; - -import static java.awt.image.BufferedImage.TYPE_INT_RGB; +import org.apache.commons.math3.complex.Complex; +import org.vitrivr.cineast.core.util.math.ZernikeMoments; +import org.vitrivr.cineast.core.util.math.functions.ZernikeBasisFunction; public final class ZernikeHelper { - /** - * Private constructor; do not instantiate! - */ - private ZernikeHelper() { + /** + * Private constructor; do not instantiate! + */ + private ZernikeHelper() { + } + + /** + * Calculates and returns the Zernike Moments of order n for the provided image. + * + * @param image BufferedImage for which Zernike Moments should be extracted. + * @param order Order up to which Zernike Moments should be calculated. + * @return ZernikeMoments for image. + */ + public static ZernikeMoments zernikeMoments(BufferedImage image, int order) { + + double[][] data = new double[image.getWidth()][image.getHeight()]; + + for (int x = 0; x < image.getWidth(); x++) { + for (int y = 0; y < image.getHeight(); y++) { + int rgb = image.getRGB(x, y); + int r = (rgb >> 16) & 0xFF; + int g = (rgb >> 8) & 0xFF; + int b = (rgb & 0xFF); + + /* Store gay level in terms of values between 0.0 and 1.0. */ + data[x][y] = (r + g + b) / (3.0 * 255.0); + } } - /** - * Calculates and returns the Zernike Moments of order n for the provided image. - * - * @param image BufferedImage for which Zernike Moments should be extracted. - * @param order Order up to which Zernike Moments should be calculated. - * @return ZernikeMoments for image. - */ - public static ZernikeMoments zernikeMoments(BufferedImage image, int order) { - - double[][] data = new double[image.getWidth()][image.getHeight()]; - - for (int x=0;x> 16) & 0xFF; - int g = (rgb >> 8) & 0xFF; - int b = (rgb & 0xFF); - - /* Store gay level in terms of values between 0.0 and 1.0. */ - data[x][y] = (r + g + b) / (3.0 * 255.0); + ZernikeMoments m = new ZernikeMoments(data); + m.compute(order); + return m; + } + + /** + * Calculates and returns a list of Zernike Moments for all shapes that have been detected in the provided image. To do so, first the image is transformed into a binary image by means of thresholding. Afterwards, contours are detected in the binary image and shapes are extracted for every detected contour. The so extracted data is then handed to a class that obtains the Zernike Moments. + * + * @param image BufferedImage for which Zernike Moments should be extracted. + * @param minimal Minimal size of the shape in terms of contour-length. Smaller shapes will be ignored. + * @param order Order up to which Zernike Moments should be calculated. + * @return List of Zernike Moments for the image. + */ + public static List zernikeMomentsForShapes(BufferedImage image, int minimal, int order) { + /* Extract the contours (shapes) from the buffered image. */ + GrayF32 input = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); + GrayU8 binary = new GrayU8(input.width, input.height); + GrayS32 label = new GrayS32(input.width, input.height); + + /* Select a global threshold using Otsu's method and apply that threshold. */ + double threshold = GThresholdImageOps.computeOtsu(input, 0, 255); + ThresholdImageOps.threshold(input, binary, (float) threshold, true); + + // remove small blobs through erosion and dilation + // The null in the input indicates that it should internally declare the work image it needs + // this is less efficient, but easier to code. + GrayU8 filtered = BinaryImageOps.erode8(binary, 1, null); + filtered = BinaryImageOps.dilate8(filtered, 1, null); + + /* Detect blobs inside the image using an 8-connect rule. */ + List contours = BinaryImageOps.contour(filtered, ConnectRule.EIGHT, label); + + List moments = new ArrayList<>(); + for (Contour contour : contours) { + for (List shape : contour.internal) { + if (shape.size() >= minimal) { + int[] bounds = ContourHelper.bounds(shape); + int w = bounds[1] - bounds[0]; + int h = bounds[3] - bounds[2]; + double[][] data = new double[w][h]; + + for (int x = 0; x < w; x++) { + for (int y = 0; y < h; y++) { + if (filtered.get(x + bounds[0], y + bounds[2]) == 1) { + data[x][y] = 0.0f; + } else { + data[x][y] = 1.0f; + } } - } + } - ZernikeMoments m = new ZernikeMoments(data); - m.compute(order); - return m; + ZernikeMoments m = new ZernikeMoments(data); + m.compute(order); + moments.add(m); + } + } } - /** - * Calculates and returns a list of Zernike Moments for all shapes that have been detected in the provided - * image. To do so, first the image is transformed into a binary image by means of thresholding. Afterwards, - * contours are detected in the binary image and shapes are extracted for every detected contour. The so - * extracted data is then handed to a class that obtains the Zernike Moments. - * - * @param image BufferedImage for which Zernike Moments should be extracted. - * @param minimal Minimal size of the shape in terms of contour-length. Smaller shapes will be ignored. - * @param order Order up to which Zernike Moments should be calculated. - * @return List of Zernike Moments for the image. - */ - public static List zernikeMomentsForShapes(BufferedImage image, int minimal, int order) { - /* Extract the contours (shapes) from the buffered image. */ - GrayF32 input = ConvertBufferedImage.convertFromSingle(image, null, GrayF32.class); - GrayU8 binary = new GrayU8(input.width,input.height); - GrayS32 label = new GrayS32(input.width,input.height); - - /* Select a global threshold using Otsu's method and apply that threshold. */ - double threshold = GThresholdImageOps.computeOtsu(input, 0, 255); - ThresholdImageOps.threshold(input, binary,(float)threshold,true); - - // remove small blobs through erosion and dilation - // The null in the input indicates that it should internally declare the work image it needs - // this is less efficient, but easier to code. - GrayU8 filtered = BinaryImageOps.erode8(binary, 1, null); - filtered = BinaryImageOps.dilate8(filtered, 1, null); - - /* Detect blobs inside the image using an 8-connect rule. */ - List contours = BinaryImageOps.contour(filtered, ConnectRule.EIGHT, label); - - List moments = new ArrayList<>(); - for (Contour contour : contours) { - for (List shape : contour.internal) { - if (shape.size() >= minimal) { - int[] bounds = ContourHelper.bounds(shape); - int w = bounds[1] - bounds[0]; - int h = bounds[3] - bounds[2]; - double[][] data = new double[w][h]; - - for (int x = 0; x < w; x++) { - for (int y = 0; y < h; y++) { - if (filtered.get(x + bounds[0],y + bounds[2]) == 1) { - data[x][y] = 0.0f; - } else { - data[x][y] = 1.0f; - } - } - } - - ZernikeMoments m = new ZernikeMoments(data); - m.compute(order); - moments.add(m); - } + return moments; + } + + /** + * Attempts at reconstructing an image from a list of complete Zernike Moments. The list must contain a complete set of complex Zernike Moments up to some arbitrary order. The moments must be ordered according to Noll's sequential index (ascending). + */ + public static BufferedImage reconstructImage(ZernikeMoments moments) { + return ZernikeHelper.reconstructImage(moments.getHeight(), moments.getHeight(), moments.getMoments()); + } + + /** + * Attempts at reconstructing an image from a list of complex Zernike Moments. The list must contain a complete set of Zernike Moments up to some arbitrary order n. The moments must be ordered according to Noll's sequential indexing scheme (ascending). + * + * @param w the width of the desired image + * @param h the height of the desired image + * @param moments List of Zernike Moments is ascending order according to Noll's index. + * @return The reconstructed image + */ + public static BufferedImage reconstructImage(final int w, final int h, final List moments) { + /* Scale x and y dimension to unit-disk. */ + final double c = -1.0; + final double d = 1.0; + + /* Prepare array for imaga data. */ + final double[][] imageData = new double[w][h]; + + double maxValue = 0.0f; + int indexFeature = 0; + for (int n = 0; indexFeature < moments.size(); ++n) { + for (int m = 0; m <= n; m++) { + if ((n - Math.abs(m)) % 2 == 0) { + final Complex moment = moments.get(indexFeature); + final ZernikeBasisFunction bf1 = new ZernikeBasisFunction(n, m); + for (int i = 0; i < w; i++) { + for (int j = 0; j < h; j++) { + Complex v = new Complex(c + (i * (d - c)) / (w - 1), d - (j * (d - c)) / (h - 1)); + Complex res = moment.multiply(bf1.value(v)); + imageData[i][j] += res.getReal(); + maxValue = Math.max(maxValue, imageData[i][j]); } + } + indexFeature++; } - - return moments; + } } - /** - * Attempts at reconstructing an image from a list of complete Zernike Moments. The list must contain - * a complete set of complex Zernike Moments up to some arbitrary order. The moments must be ordered according - * to Noll's sequential index (ascending). - */ - public static BufferedImage reconstructImage(ZernikeMoments moments) { - return ZernikeHelper.reconstructImage(moments.getHeight(), moments.getHeight(), moments.getMoments()); + BufferedImage image = new BufferedImage(w, h, TYPE_INT_RGB); + for (int x = 1; x < w; x++) { + for (int y = 1; y < h; y++) { + int i = (int) (255.0 * (imageData[x - 1][y - 1]) / (maxValue)); + int rgb = ((255 & 0xFF) << 24) | ((i & 0xFF) << 16) | ((i & 0xFF) << 8) | ((i & 0xFF) << 0); + image.setRGB(x, y, rgb); + } } - /** - * Attempts at reconstructing an image from a list of complex Zernike Moments. The list must contain - * a complete set of Zernike Moments up to some arbitrary order n. The moments must be ordered - * according to Noll's sequential indexing scheme (ascending). - * - * @param w the width of the desired image - * @param h the height of the desired image - * @param moments List of Zernike Moments is ascending order according to Noll's index. - * @return The reconstructed image - */ - public static BufferedImage reconstructImage(final int w, final int h, final List moments) { - /* Scale x and y dimension to unit-disk. */ - final double c = -1.0; - final double d = 1.0; - - /* Prepare array for imaga data. */ - final double[][] imageData = new double[w][h]; - - double maxValue = 0.0f; - int indexFeature = 0; - for (int n=0; indexFeature < moments.size(); ++n) { - for (int m=0;m<=n;m++) { - if((n-Math.abs(m))%2 == 0) { - final Complex moment = moments.get(indexFeature); - final ZernikeBasisFunction bf1 = new ZernikeBasisFunction(n, m); - for (int i = 0; i < w; i++) { - for (int j = 0; j < h; j++) { - Complex v = new Complex(c+(i*(d-c))/(w-1), d-(j*(d-c))/(h-1)); - Complex res = moment.multiply(bf1.value(v)); - imageData[i][j] += res.getReal(); - maxValue = Math.max(maxValue, imageData[i][j]); - } - } - indexFeature++; - } - } - } - - BufferedImage image = new BufferedImage(w, h, TYPE_INT_RGB); - for (int x = 1; x < w; x++) { - for (int y = 1; y < h; y++) { - int i = (int)(255.0*(imageData[x-1][y-1] )/(maxValue)); - int rgb = ((255 & 0xFF) << 24) | ((i & 0xFF) << 16) | ((i & 0xFF) << 8) | ((i & 0xFF) << 0); - image.setRGB(x,y,rgb); - } - } - - return image; - } + return image; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JacksonJsonProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JacksonJsonProvider.java index 99a3b651c..d8fe22513 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JacksonJsonProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JacksonJsonProvider.java @@ -7,30 +7,29 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.util.LogHelper; - -import javax.annotation.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Path; +import javax.annotation.Nullable; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.util.LogHelper; /** * Provides JSON deserialization capabilities by means of Jackson Databind library. - * */ public class JacksonJsonProvider implements JsonReader, JsonWriter { /** - * Jackson ObjectMapper instance used to map to/from objects. As this class is thread safe it can - * be shared and therefore be static! + * Jackson ObjectMapper instance used to map to/from objects. As this class is thread safe it can be shared and therefore be static! */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** Logger used to log errors. */ + /** + * Logger used to log errors. + */ private static final Logger LOGGER = LogManager.getLogger(); /* @@ -111,8 +110,7 @@ private static void logIOExceptionOfReadWithMessage(IOException e, String jsonTy } /** - * Takes a Java Object (usually a POJO) and tries to serialize it into a JSON. If serialization - * fails for some reason, this method should return JSON_EMPTY; + * Takes a Java Object (usually a POJO) and tries to serialize it into a JSON. If serialization fails for some reason, this method should return JSON_EMPTY; */ @Override public String toJson(Object object) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonReader.java index f2f5655f0..bf041c208 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonReader.java @@ -1,33 +1,30 @@ package org.vitrivr.cineast.core.util.json; -import javax.annotation.Nullable; import java.io.File; +import javax.annotation.Nullable; /** - * Wraps the Json to Object deserialization so as to make sure that it can be provided - * independently of a concrete library. - * + * Wraps the Json to Object deserialization so as to make sure that it can be provided independently of a concrete library. */ public interface JsonReader { + /** - * Tries to deserialize a JSON string into a object of the provided class. On error - * this method returns null. Exceptions must be handled by the implementation! + * Tries to deserialize a JSON string into a object of the provided class. On error this method returns null. Exceptions must be handled by the implementation! * * @param jsonString String to deserialize. - * @param c Class to which the string should be deserialized. - * @param Type of Object that should be returned. + * @param c Class to which the string should be deserialized. + * @param Type of Object that should be returned. * @return Object of type T on success or null otherwise. */ @Nullable T toObject(String jsonString, Class c); /** - * Tries to deserialize a JSON file into a object of the provided class. On error - * this method returns null. Exceptions must be handled by the implementation! + * Tries to deserialize a JSON file into a object of the provided class. On error this method returns null. Exceptions must be handled by the implementation! * * @param json File to deserialize. - * @param c Class to which the string should be deserialized. - * @param Type of Object that should be returned. + * @param c Class to which the string should be deserialized. + * @param Type of Object that should be returned. * @return Object of type T on success or null otherwise. */ @Nullable diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java index af8fae1c9..2f3170fa5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/json/JsonWriter.java @@ -1,17 +1,14 @@ package org.vitrivr.cineast.core.util.json; /** - * Wraps the Object to JSON serialization so as to make sure that it can be provided - * independently of a concrete library. - * + * Wraps the Object to JSON serialization so as to make sure that it can be provided independently of a concrete library. */ public interface JsonWriter { - String JSON_EMPTY = "{}"; + String JSON_EMPTY = "{}"; - /** - * Takes a Java Object (usually a POJO) and tries to serialize it into a JSON. If serialization - * fails for some reason, this method should return JSON_EMPTY; - */ - String toJson(Object object); + /** + * Takes a Java Object (usually a POJO) and tries to serialize it into a JSON. If serialization fails for some reason, this method should return JSON_EMPTY; + */ + String toJson(Object object); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/MathConstants.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/MathConstants.java index 9a30eac53..53b668cae 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/MathConstants.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/MathConstants.java @@ -2,32 +2,41 @@ public final class MathConstants { - /** Definition of the golden ratio PHI. */ - public static final double PHI = ((1.0+Math.sqrt(5.0))/2.0); - - /** Square-root of three.*/ - public static final double SQRT3 = Math.sqrt(3); - - - /** Defines the vertices of a regular Cube. */ - public static final double[][] VERTICES_3D_CUBE = { - {1,1,1},{-1,-1,-1},{1,-1,-1},{-1,-1,1}, - {-1,1,-1},{-1,1,1},{1,-1,1},{1,1,-1} - }; - - /** Defines the vertices of a regular Dodecahedron. */ - public static final double[][] VERTICES_3D_DODECAHEDRON = { - {1,1,1},{-1,-1,-1},{1,-1,-1},{-1,-1,1}, - {-1,1,-1},{-1,1,1},{1,-1,1},{1,1,-1}, - {0,1/PHI,PHI},{0,-1/PHI,PHI},{0,1/PHI,-PHI},{0,-1/PHI,-PHI}, - {1/PHI,PHI,0},{-1/PHI,PHI,0},{1/PHI,-PHI,0},{-1/PHI,-PHI,0}, - { PHI,0,1/PHI},{- PHI,0,1/PHI},{ PHI,0,-1/PHI},{-PHI,0,-1/PHI} - }; - - /** - * Private constructor; no instantiation possible! - */ - private MathConstants() { - - } + + /** + * Definition of the golden ratio PHI. + */ + public static final double PHI = ((1.0 + Math.sqrt(5.0)) / 2.0); + + /** + * Square-root of three. + */ + public static final double SQRT3 = Math.sqrt(3); + + + /** + * Defines the vertices of a regular Cube. + */ + public static final double[][] VERTICES_3D_CUBE = { + {1, 1, 1}, {-1, -1, -1}, {1, -1, -1}, {-1, -1, 1}, + {-1, 1, -1}, {-1, 1, 1}, {1, -1, 1}, {1, 1, -1} + }; + + /** + * Defines the vertices of a regular Dodecahedron. + */ + public static final double[][] VERTICES_3D_DODECAHEDRON = { + {1, 1, 1}, {-1, -1, -1}, {1, -1, -1}, {-1, -1, 1}, + {-1, 1, -1}, {-1, 1, 1}, {1, -1, 1}, {1, 1, -1}, + {0, 1 / PHI, PHI}, {0, -1 / PHI, PHI}, {0, 1 / PHI, -PHI}, {0, -1 / PHI, -PHI}, + {1 / PHI, PHI, 0}, {-1 / PHI, PHI, 0}, {1 / PHI, -PHI, 0}, {-1 / PHI, -PHI, 0}, + {PHI, 0, 1 / PHI}, {-PHI, 0, 1 / PHI}, {PHI, 0, -1 / PHI}, {-PHI, 0, -1 / PHI} + }; + + /** + * Private constructor; no instantiation possible! + */ + private MathConstants() { + + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/ZernikeMoments.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/ZernikeMoments.java index 604ad3810..b294da778 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/ZernikeMoments.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/ZernikeMoments.java @@ -1,131 +1,133 @@ package org.vitrivr.cineast.core.util.math; -import org.apache.commons.math3.complex.Complex; -import org.vitrivr.cineast.core.util.math.functions.ZernikeBasisFunction; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.math3.complex.Complex; +import org.vitrivr.cineast.core.util.math.functions.ZernikeBasisFunction; /** * This class can be used to compute Zernike Moments. - * */ public final class ZernikeMoments { - /** List of calculated Zernike Moments. */ - private final List moments = new ArrayList<>(); - - /** 2D data for which Zernike moment should be calculated. */ - private final double[][] f; - - /** Width of the original data (x-dimension). */ - private final int width; - - /** Height of the original data (y-dimension). */ - private final int height; - - /** - * Default constructor for Zernike Moment. - * - * @param f 2D array containing the data for which the moments should be calculated. - */ - public ZernikeMoments(double[][] f) { - this.f = f; - this.width = f.length; - this.height = f[0].length; - } - - /** - * Computes the Zernike Moments up to order n. - * - * @param n Maximum order n. - */ - public void compute(int n) { - this.moments.clear(); - this.moments.addAll(ZernikeMoments.calculateZernikeMoments(n, this.f, this.width, this.height)); + /** + * List of calculated Zernike Moments. + */ + private final List moments = new ArrayList<>(); + + /** + * 2D data for which Zernike moment should be calculated. + */ + private final double[][] f; + + /** + * Width of the original data (x-dimension). + */ + private final int width; + + /** + * Height of the original data (y-dimension). + */ + private final int height; + + /** + * Default constructor for Zernike Moment. + * + * @param f 2D array containing the data for which the moments should be calculated. + */ + public ZernikeMoments(double[][] f) { + this.f = f; + this.width = f.length; + this.height = f[0].length; + } + + /** + * Computes the Zernike Moments up to order n. + * + * @param n Maximum order n. + */ + public void compute(int n) { + this.moments.clear(); + this.moments.addAll(ZernikeMoments.calculateZernikeMoments(n, this.f, this.width, this.height)); + } + + /** + * Getter for the moments array. Must be computed prior to calling this method. Otherwise the list will be empty. + * + * @return List of Zernike Moments. + */ + public List getMoments() { + return this.moments; + } + + /** + * Compute Zernike moments at specified order. + * + * @param w Width of the bounding box of the shape. + * @param h Height of the bounding box of the shape. + * @param n 1st order of the moment. + * @param m 2nd order of the moment. + * @return Zernike moment of data in f[][]. + */ + public static Complex calculateZernikeMoment(double[][] f, int w, int h, int n, int m) { + int diff = n - Math.abs(m); + if ((n < 0) || (Math.abs(m) > n) || (diff % 2 != 0)) { + throw new IllegalArgumentException("zer_mom: n=" + n + ", m=" + m + ", n-|m|=" + diff); } - /** - * Getter for the moments array. Must be computed prior - * to calling this method. Otherwise the list will be empty. - * - * @return List of Zernike Moments. - */ - public List getMoments() { - return this.moments; - } - - /** - * Compute Zernike moments at specified order. - * - * @param w Width of the bounding box of the shape. - * @param h Height of the bounding box of the shape. - * @param n 1st order of the moment. - * @param m 2nd order of the moment. - * - * @return Zernike moment of data in f[][]. - */ - public static Complex calculateZernikeMoment(double[][] f, int w, int h, int n, int m){ - int diff = n-Math.abs(m); - if ((n<0) || (Math.abs(m) > n) || (diff%2!=0)){ - throw new IllegalArgumentException("zer_mom: n="+n+", m="+m+", n-|m|="+diff); - } + final double c = -1; + final double d = 1; - final double c = -1; - final double d = 1; - - - ZernikeBasisFunction zernike = new ZernikeBasisFunction(n,m); - Complex res = new Complex(0.0, 0.0); - for (int i=0;i calculateZernikeMoments(int order, double[][] f, int w, int h){ - ArrayList list = new ArrayList<>(order); - for(int n=0; n<=order; n++){ - for(int m=0; m<=n; m++){ - if((n-Math.abs(m))%2 == 0){ - Complex v = calculateZernikeMoment(f,w,h,n,m); - list.add(v); - } - } + return res.multiply((n + 1) / Math.PI); + } + + /** + * Compute the set of Zernike's moments up to the specified order. + * + * @param order Maximum order n (1st order). + * @param w Width of the bounding box of the shape. + * @param h Height of the bounding box of the shape. + * @return List of Zernike moments. + */ + public static List calculateZernikeMoments(int order, double[][] f, int w, int h) { + ArrayList list = new ArrayList<>(order); + for (int n = 0; n <= order; n++) { + for (int m = 0; m <= n; m++) { + if ((n - Math.abs(m)) % 2 == 0) { + Complex v = calculateZernikeMoment(f, w, h, n, m); + list.add(v); } - return list; - } - - /** - * Getter for width of the original image. - * - * @return Width of the original image. - */ - public int getWidth() { - return width; - } - - /** - * Getter for height of the original image. - * - * @return Height of the original image. - */ - public int getHeight() { - return height; + } } + return list; + } + + /** + * Getter for width of the original image. + * + * @return Width of the original image. + */ + public int getWidth() { + return width; + } + + /** + * Getter for height of the original image. + * + * @return Height of the original image. + */ + public int getHeight() { + return height; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/AssociatedLegendrePolynomial.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/AssociatedLegendrePolynomial.java index 0109671ca..54f0cdf2d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/AssociatedLegendrePolynomial.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/AssociatedLegendrePolynomial.java @@ -8,84 +8,85 @@ public final class AssociatedLegendrePolynomial implements UnivariateFunction { - /** Legendre used to generate base values for the associated Legendre polynomial. That polynomial - * is the m-th derivative of the l-th Legendre polynomial. - */ - private final PolynomialFunction legendre; + /** + * Legendre used to generate base values for the associated Legendre polynomial. That polynomial is the m-th derivative of the l-th Legendre polynomial. + */ + private final PolynomialFunction legendre; - /** Degree of the associated Legendre polynomial */ - private final int l; + /** + * Degree of the associated Legendre polynomial + */ + private final int l; - /** Order of the associated Legendre polynomial */ - private final int m; + /** + * Order of the associated Legendre polynomial + */ + private final int m; - /** Sign of the polynomial which is only determined by m. */ - private final double sign; + /** + * Sign of the polynomial which is only determined by m. + */ + private final double sign; - /** - * Constructor for the AssociatedLegendrePolynomial class. - * - * @param l Degree of the associated Legendre polynomial - * @param m Order of the associated Legendre polynomial - */ - public AssociatedLegendrePolynomial(int l, int m) { + /** + * Constructor for the AssociatedLegendrePolynomial class. + * + * @param l Degree of the associated Legendre polynomial + * @param m Order of the associated Legendre polynomial + */ + public AssociatedLegendrePolynomial(int l, int m) { - /* Make some basic, arithmetic checks. */ - if (m > l) { - throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m > l!"); - } - if (m < 0) { - throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m < 0!"); - } - if (l < 0) { - throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m < 0!"); - } - - /* Find m-th derivative of Legendre Polynomial of degree l. */ - PolynomialFunction fkt = PolynomialsUtils.createLegendrePolynomial(l); - for (int i = 0; i < m; i++) { - fkt = fkt.polynomialDerivative(); - } - this.legendre = fkt; - - /* Determine sign. */ - this.sign = Math.pow(-1,m); - this.m = m; - this.l = l; + /* Make some basic, arithmetic checks. */ + if (m > l) { + throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m > l!"); } - - /** - * Compute the value of the function. - * - * @param x Point at which the function value should be computed. - * @return the value of the function. - * @throws IllegalArgumentException when the activated method itself can - * ascertain that a precondition, specified in the API expressed at the - * level of the activated method, has been violated. - * When Commons Math throws an {@code IllegalArgumentException}, it is - * usually the consequence of checking the actual parameters passed to - * the method. - */ - @Override - public final double value(double x) { - return this.sign * (FastMath.pow(1.0-FastMath.pow(x,2.0),m/2.0) * legendre.value(x)); + if (m < 0) { + throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m < 0!"); } - - /** - * Returns the value of L for the AssociatedLegendrePolynomial - * - * @return L - */ - public final int getL() { - return this.l; + if (l < 0) { + throw new IllegalArgumentException("Associated Legendre Polynomials are defined for 0 <= m <= l. You provided m < 0!"); } - /** - * Returns the value of M for the AssociatedLegendrePolynomial - * - * @return M - */ - public final int getM() { - return this.m; + /* Find m-th derivative of Legendre Polynomial of degree l. */ + PolynomialFunction fkt = PolynomialsUtils.createLegendrePolynomial(l); + for (int i = 0; i < m; i++) { + fkt = fkt.polynomialDerivative(); } + this.legendre = fkt; + + /* Determine sign. */ + this.sign = Math.pow(-1, m); + this.m = m; + this.l = l; + } + + /** + * Compute the value of the function. + * + * @param x Point at which the function value should be computed. + * @return the value of the function. + * @throws IllegalArgumentException when the activated method itself can ascertain that a precondition, specified in the API expressed at the level of the activated method, has been violated. When Commons Math throws an {@code IllegalArgumentException}, it is usually the consequence of checking the actual parameters passed to the method. + */ + @Override + public final double value(double x) { + return this.sign * (FastMath.pow(1.0 - FastMath.pow(x, 2.0), m / 2.0) * legendre.value(x)); + } + + /** + * Returns the value of L for the AssociatedLegendrePolynomial + * + * @return L + */ + public final int getL() { + return this.l; + } + + /** + * Returns the value of M for the AssociatedLegendrePolynomial + * + * @return M + */ + public final int getM() { + return this.m; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java index 5039b327f..8f32fa296 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/SphericalHarmonicsFunction.java @@ -7,86 +7,93 @@ public class SphericalHarmonicsFunction { - /** Degree of the Spherical Harmonics function (l >= 0) */ - private final int l; - - /** Order of the Spherical Harmonics function (-l >= m >= l) */ - private final int m; - - /** Normalization factor for spherical harmonic function. */ - private final double Nlm; - - /** Associated Legendre Polynomial used to calculate the Spherical Harmonic function. */ - private final AssociatedLegendrePolynomial legendre; - - /** - * Constructor for the SphericalHarmonicsFunction class. - * - * @param l Degree of the Spherical Harmonics function - * @param m Order of the Spherical Harmonics function - * @throws IllegalArgumentException if l or m do not satisfy the contraints. - */ - public SphericalHarmonicsFunction(int l, int m) { + /** + * Degree of the Spherical Harmonics function (l >= 0) + */ + private final int l; + + /** + * Order of the Spherical Harmonics function (-l >= m >= l) + */ + private final int m; + + /** + * Normalization factor for spherical harmonic function. + */ + private final double Nlm; + + /** + * Associated Legendre Polynomial used to calculate the Spherical Harmonic function. + */ + private final AssociatedLegendrePolynomial legendre; + + /** + * Constructor for the SphericalHarmonicsFunction class. + * + * @param l Degree of the Spherical Harmonics function + * @param m Order of the Spherical Harmonics function + * @throws IllegalArgumentException if l or m do not satisfy the contraints. + */ + public SphericalHarmonicsFunction(int l, int m) { + + if (l < 0) { + throw new IllegalArgumentException("Spherical harmonics functions are not defined for l < 0."); + } + if (Math.abs(m) > l) { + throw new IllegalArgumentException("Spherical harmonics functions are not defined for |m| > l."); + } - if (l < 0) { - throw new IllegalArgumentException("Spherical harmonics functions are not defined for l < 0."); + this.l = l; + this.m = m; + + /* Calculate constants. */ + this.Nlm = SphericalHarmonicsFunction.getFactor(l, m); + + /* Instantiate associated legendre polynomial. */ + this.legendre = new AssociatedLegendrePolynomial(l, Math.abs(m)); + } + + + /** + * Compute the value of the function. + * + * @param theta Point at which the function value should be computed. + * @param phi Point at which the function value should be computed. + * @return the complex value of the function. + */ + public Complex value(final double theta, final double phi) { + double r = this.Nlm * this.legendre.value(FastMath.cos(theta)); + double arg = this.m * phi; + return new Complex(r * FastMath.cos(arg), r * FastMath.sin(arg)); + } + + /** + * Calculates and returns the normalisation factor for the Spherical Harmonics Function + * + * @param l Order + */ + public static double getFactor(int l, int m) { + return FastMath.sqrt(((2 * l + 1) / (4 * Math.PI)) * ((double) CombinatoricsUtils.factorial(l - FastMath.abs(m)) / (double) CombinatoricsUtils.factorial(l + FastMath.abs(m)))); + } + + /** + * Calculates and returns the number of coefficients that are expected for all spherical harmonics up to max_l. + * + * @param max_l The maximum harmonic to consider. + */ + public static int numberOfCoefficients(int max_l, boolean onesided) { + int number = 0; + for (int l = 0; l <= max_l; l++) { + if (onesided) { + for (int m = 0; m <= l; m++) { + number += 1; } - if (Math.abs(m) > l) { - throw new IllegalArgumentException("Spherical harmonics functions are not defined for |m| > l."); + } else { + for (int m = -l; m <= l; m++) { + number += 1; } - - this.l = l; - this.m = m; - - /* Calculate constants. */ - this.Nlm = SphericalHarmonicsFunction.getFactor(l,m); - - /* Instantiate associated legendre polynomial. */ - this.legendre = new AssociatedLegendrePolynomial(l,Math.abs(m)); + } } - - - /** - * Compute the value of the function. - * - * @param theta Point at which the function value should be computed. - * @param phi Point at which the function value should be computed. - * @return the complex value of the function. - */ - public Complex value(final double theta, final double phi) { - double r = this.Nlm * this.legendre.value(FastMath.cos(theta)); - double arg = this.m * phi; - return new Complex(r * FastMath.cos(arg), r * FastMath.sin(arg)); - } - - /** - * Calculates and returns the normalisation factor for the Spherical Harmonics Function - * - * @param l Order - */ - public static double getFactor(int l, int m) { - return FastMath.sqrt(((2*l + 1)/(4*Math.PI)) * ((double)CombinatoricsUtils.factorial(l-FastMath.abs(m)) / (double)CombinatoricsUtils.factorial(l+FastMath.abs(m)))); - } - - /** - * Calculates and returns the number of coefficients that are expected for all spherical harmonics - * up to max_l. - * - * @param max_l The maximum harmonic to consider. - */ - public static int numberOfCoefficients(int max_l, boolean onesided) { - int number = 0; - for (int l=0; l<=max_l; l++) { - if (onesided) { - for (int m=0; m<=l; m++) { - number += 1; - } - } else { - for (int m=-l; m<=l; m++) { - number += 1; - } - } - } - return number; - } + return number; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java index 573f8793a..c3275452d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/ZernikeBasisFunction.java @@ -7,102 +7,105 @@ import org.vitrivr.cineast.core.util.math.functions.interfaces.UnivariateComplexFunction; /** - * This class represents a Zernike Basis Function of order m,n. These functions can be - * used to construct a Zernike Polynomial. - * + * This class represents a Zernike Basis Function of order m,n. These functions can be used to construct a Zernike Polynomial. */ public class ZernikeBasisFunction implements UnivariateComplexFunction { - /** Radial polynomial (R_nm) used to calculate the ZernikeBasisFunction. */ - private final PolynomialFunction radialPolynomial; + /** + * Radial polynomial (R_nm) used to calculate the ZernikeBasisFunction. + */ + private final PolynomialFunction radialPolynomial; - /** 1st moment of the radial polynomial (order). */ - private final int n; + /** + * 1st moment of the radial polynomial (order). + */ + private final int n; - /** 2nd moment of the radial polynomial (repetition). */ - private final int m; + /** + * 2nd moment of the radial polynomial (repetition). + */ + private final int m; - /** - * Constructs a new ZernikeBasisFunction given the 1st and 2nd moment used - * to construct the radial polynomial associated with it. - * - * @param n 1st moment of the Zernike Basis Function. - * @param m 2nd moment of the Zernike Basis Function. - * @throws IllegalArgumentException If |m| > n, m <'0 or (n-|m| % 2) = 1 - */ - public ZernikeBasisFunction(int n, int m) { - if (n < 0) { - throw new IllegalArgumentException("Zernike Basis Functions are only defined for n > 0"); - } - if ((n-Math.abs(m)) < 0) { - throw new IllegalArgumentException("Zernike Basis Functions are only defined for |m|<=n"); - } - if ((n-Math.abs(m)) % 2 != 0) { - throw new IllegalArgumentException("Zernike Basis Functions are only defined for (n-|m| % 2) = 0."); - } - - this.radialPolynomial = PolynomialFunctionFactory.createRadialPolynomial(n,m); - this.n = n; - this.m = m; + /** + * Constructs a new ZernikeBasisFunction given the 1st and 2nd moment used to construct the radial polynomial associated with it. + * + * @param n 1st moment of the Zernike Basis Function. + * @param m 2nd moment of the Zernike Basis Function. + * @throws IllegalArgumentException If |m| > n, m <'0 or (n-|m| % 2) = 1 + */ + public ZernikeBasisFunction(int n, int m) { + if (n < 0) { + throw new IllegalArgumentException("Zernike Basis Functions are only defined for n > 0"); } - - /** - * Computes the value of the function. - * - * @param value Point at which the function value should be computed. - * @return the value of the function. - */ - @Override - public Complex value(Complex value) { - /* Check that length of r is smaller or equal 1. */ - if (value.abs() > 1.0) { - return new Complex(0.0, 0.0); - } - - /* Calculate value of Zernike Basis Function at point. */ - double r = this.radialPolynomial.value(value.abs()); - double arg = value.getArgument() * this.m; - double a = r * FastMath.cos(arg); - double b = r * FastMath.sin(arg); - return new Complex(a, b); + if ((n - Math.abs(m)) < 0) { + throw new IllegalArgumentException("Zernike Basis Functions are only defined for |m|<=n"); + } + if ((n - Math.abs(m)) % 2 != 0) { + throw new IllegalArgumentException("Zernike Basis Functions are only defined for (n-|m| % 2) = 0."); } - /** - * Getter for the radial polynomial. - * - * @return PolynomialFunction - */ - public final PolynomialFunction getRadialPolynomial() { - return this.radialPolynomial; + this.radialPolynomial = PolynomialFunctionFactory.createRadialPolynomial(n, m); + this.n = n; + this.m = m; + } + + /** + * Computes the value of the function. + * + * @param value Point at which the function value should be computed. + * @return the value of the function. + */ + @Override + public Complex value(Complex value) { + /* Check that length of r is smaller or equal 1. */ + if (value.abs() > 1.0) { + return new Complex(0.0, 0.0); } - /** - * Custom equals implementation. - */ - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + /* Calculate value of Zernike Basis Function at point. */ + double r = this.radialPolynomial.value(value.abs()); + double arg = value.getArgument() * this.m; + double a = r * FastMath.cos(arg); + double b = r * FastMath.sin(arg); + return new Complex(a, b); + } - ZernikeBasisFunction that = (ZernikeBasisFunction) o; + /** + * Getter for the radial polynomial. + * + * @return PolynomialFunction + */ + public final PolynomialFunction getRadialPolynomial() { + return this.radialPolynomial; + } - if (n != that.n) { - return false; - } - return m == that.m; + /** + * Custom equals implementation. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; } - /** - * Custom hashCode() implementation. - */ - @Override - public int hashCode() { - int result = n; - result = 31 * result + m; - return result; + ZernikeBasisFunction that = (ZernikeBasisFunction) o; + + if (n != that.n) { + return false; } + return m == that.m; + } + + /** + * Custom hashCode() implementation. + */ + @Override + public int hashCode() { + int result = n; + result = 31 * result + m; + return result; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/factories/PolynomialFunctionFactory.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/factories/PolynomialFunctionFactory.java index 4443db150..894eb8b25 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/factories/PolynomialFunctionFactory.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/factories/PolynomialFunctionFactory.java @@ -1,59 +1,56 @@ package org.vitrivr.cineast.core.util.math.functions.factories; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.math3.analysis.polynomials.PolynomialFunction; import org.apache.commons.math3.util.CombinatoricsUtils; import org.apache.commons.math3.util.FastMath; -import java.util.concurrent.ConcurrentHashMap; - /** - * This class can be used to generate different types of polynomials. It leverages the PolynomialFunction - * class provided by Apache Commons Math library. - * + * This class can be used to generate different types of polynomials. It leverages the PolynomialFunction class provided by Apache Commons Math library. */ public final class PolynomialFunctionFactory { - /** - * Cache for Zernike Coefficients. - */ - private final static ConcurrentHashMap RADIAL_FUNCTION_CACHE = new ConcurrentHashMap<>(); - - /** - * Creates and returns a new radial polynomial (R_nm) given two moments. - * - * @param n 1st moment (order) of the radial polynomial. - * @param m 2nd moment (repetition) of the radial polynomial. - * @return PolynomialFunction representing R_nm - * @throws ArithmeticException If orders are to large and calculation of binomial coefficients fail. - */ - public static PolynomialFunction createRadialPolynomial(final int n, int m) { - m = Math.abs(m); /* Make sure that m is positive. */ - String id = n + "-" + m; /* Construct ID for cache lookup. */ - - /* Try to retrieve the function from cache. */ - if (RADIAL_FUNCTION_CACHE.containsKey(id)) { - return RADIAL_FUNCTION_CACHE.get(id); - } - - /* Initialize coefficients. */ - double[] coefficients = new double[n + 1]; - - /* Now check if Polynomial 0 (for n-|m| = odd) .*/ - if ((n - m) % 2 != 0) { - return new PolynomialFunction(coefficients); /* If (n-m) != even, return 0 function. */ - } - int s_max = (n - m) / 2; - - double sign = -1.0; - for (int s = 0; s <= s_max; ++s) { - int position = n - 2 * s; - long a = CombinatoricsUtils.binomialCoefficient(n-s, s); - long b = CombinatoricsUtils.binomialCoefficient(n-2*s, s_max - s); - coefficients[position] = (FastMath.pow(sign,s) * a * b); - } - - PolynomialFunction function = new PolynomialFunction(coefficients); - RADIAL_FUNCTION_CACHE.put(id, function); - return function; + /** + * Cache for Zernike Coefficients. + */ + private final static ConcurrentHashMap RADIAL_FUNCTION_CACHE = new ConcurrentHashMap<>(); + + /** + * Creates and returns a new radial polynomial (R_nm) given two moments. + * + * @param n 1st moment (order) of the radial polynomial. + * @param m 2nd moment (repetition) of the radial polynomial. + * @return PolynomialFunction representing R_nm + * @throws ArithmeticException If orders are to large and calculation of binomial coefficients fail. + */ + public static PolynomialFunction createRadialPolynomial(final int n, int m) { + m = Math.abs(m); /* Make sure that m is positive. */ + String id = n + "-" + m; /* Construct ID for cache lookup. */ + + /* Try to retrieve the function from cache. */ + if (RADIAL_FUNCTION_CACHE.containsKey(id)) { + return RADIAL_FUNCTION_CACHE.get(id); + } + + /* Initialize coefficients. */ + double[] coefficients = new double[n + 1]; + + /* Now check if Polynomial 0 (for n-|m| = odd) .*/ + if ((n - m) % 2 != 0) { + return new PolynomialFunction(coefficients); /* If (n-m) != even, return 0 function. */ } + int s_max = (n - m) / 2; + + double sign = -1.0; + for (int s = 0; s <= s_max; ++s) { + int position = n - 2 * s; + long a = CombinatoricsUtils.binomialCoefficient(n - s, s); + long b = CombinatoricsUtils.binomialCoefficient(n - 2 * s, s_max - s); + coefficients[position] = (FastMath.pow(sign, s) * a * b); + } + + PolynomialFunction function = new PolynomialFunction(coefficients); + RADIAL_FUNCTION_CACHE.put(id, function); + return function; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/interfaces/UnivariateComplexFunction.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/interfaces/UnivariateComplexFunction.java index 4b92fdc0d..a0527a4af 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/interfaces/UnivariateComplexFunction.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/math/functions/interfaces/UnivariateComplexFunction.java @@ -3,16 +3,15 @@ import org.apache.commons.math3.complex.Complex; /** - * This interface can be implemented by classes that implement a univariate complex function, i.e. a function - * that take a complex argument and returns a complex result. - * + * This interface can be implemented by classes that implement a univariate complex function, i.e. a function that take a complex argument and returns a complex result. */ public interface UnivariateComplexFunction { - /** - * Computes the value of the function. - * - * @param value Point at which the function value should be computed. - * @return the value of the function. - */ - public Complex value(Complex value); + + /** + * Computes the value of the function. + * + * @param value Point at which the function value should be computed. + * @return the value of the function. + */ + public Complex value(Complex value); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java index 3b9289bca..affc5baa1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshColoringUtil.java @@ -1,48 +1,49 @@ package org.vitrivr.cineast.core.util.mesh; +import java.awt.Color; import org.joml.Vector3f; import org.joml.Vector3fc; import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.data.m3d.WritableMesh; -import java.awt.*; - public final class MeshColoringUtil { - /** - * Private constructor; cannot be instantiated. - */ - private MeshColoringUtil() {} - - /** - * Adds colours to the vertices proportional to their distance to the barycenter. - * - * @param mesh Mesh that needs coloring. - */ - public static void color(WritableMesh mesh) { - Vector3fc center = mesh.barycenter(); - Mesh.Vertex farthestVertex = MeshMathUtil.farthestVertex(mesh, center); - float ds_max = center.distance(farthestVertex.getPosition()); - for (int i=0; i * [1] Vranić, D. and D. S. (n.d.). 3D Model Retrieval. - * */ public final class MeshMathUtil { - /** - * Private constructor; cannot be instantiated. - */ - private MeshMathUtil() {} + /** + * Private constructor; cannot be instantiated. + */ + private MeshMathUtil() { + } - /** - * Returns the vertex from a mesh that is farthest away from a given point. - * - * @param mesh Mesh from which the farthest vertex should be selected. - * @param point Point to which the distance should be calculated. - * @return Coordinates of the vertex that is farthest to the provided point. - */ - public static Mesh.Vertex farthestVertex(ReadableMesh mesh, Vector3fc point) { - List vertices = mesh.getVertices(); - Mesh.Vertex max = vertices.get(0); - float dsq_max = point.distanceSquared(max.getPosition()); - for (Mesh.Vertex v : vertices) { - float dsq = point.distanceSquared(v.getPosition()); - if (dsq > dsq_max) { - dsq_max = dsq; - max = v; - } - } - return max; + /** + * Returns the vertex from a mesh that is farthest away from a given point. + * + * @param mesh Mesh from which the farthest vertex should be selected. + * @param point Point to which the distance should be calculated. + * @return Coordinates of the vertex that is farthest to the provided point. + */ + public static Mesh.Vertex farthestVertex(ReadableMesh mesh, Vector3fc point) { + List vertices = mesh.getVertices(); + Mesh.Vertex max = vertices.get(0); + float dsq_max = point.distanceSquared(max.getPosition()); + for (Mesh.Vertex v : vertices) { + float dsq = point.distanceSquared(v.getPosition()); + if (dsq > dsq_max) { + dsq_max = dsq; + max = v; + } } + return max; + } - /** - * Returns the vertex from a mesh that is closest to a given point. - * - * @param mesh Mesh from which the closest vertex should be selected. - * @param point Point to which the distance should be calculated. - * @return Coordinates of the vertex that is closest to the provided point. - */ - public static Mesh.Vertex closestVertex(ReadableMesh mesh, Vector3fc point) { - List vertices = mesh.getVertices(); - Mesh.Vertex min = vertices.get(0); - float dsq_min = point.distanceSquared(min.getPosition()); - for (Mesh.Vertex v : mesh.getVertices()) { - float dsq = point.distanceSquared(v.getPosition()); - if (dsq < dsq_min) { - dsq_min = dsq; - min = v; - } - } - return min; + /** + * Returns the vertex from a mesh that is closest to a given point. + * + * @param mesh Mesh from which the closest vertex should be selected. + * @param point Point to which the distance should be calculated. + * @return Coordinates of the vertex that is closest to the provided point. + */ + public static Mesh.Vertex closestVertex(ReadableMesh mesh, Vector3fc point) { + List vertices = mesh.getVertices(); + Mesh.Vertex min = vertices.get(0); + float dsq_min = point.distanceSquared(min.getPosition()); + for (Mesh.Vertex v : mesh.getVertices()) { + float dsq = point.distanceSquared(v.getPosition()); + if (dsq < dsq_min) { + dsq_min = dsq; + min = v; + } } + return min; + } - /** - * Calculates the center of mass (barycenter) of a polyhedral mesh by obtaining - * the mean of the Mesh's face centroids weighted by the area of the respective face as - * described in [1]. - * - * @param mesh The mesh for which the barycenter should be calculated. - * @return Coordinates of the barycenter. - */ - public static Vector3f barycenter(ReadableMesh mesh) { - Vector3f barycenter = new Vector3f(0f,0f,0f); - double total = 0.0; - for (Mesh.Face face : mesh.getFaces()) { - double area = face.area(); - if (area > 0.0) { - barycenter.add(face.centroid().mul((float)area)); - total += area; - } - } - barycenter.div((float)total); - return barycenter; + /** + * Calculates the center of mass (barycenter) of a polyhedral mesh by obtaining the mean of the Mesh's face centroids weighted by the area of the respective face as described in [1]. + * + * @param mesh The mesh for which the barycenter should be calculated. + * @return Coordinates of the barycenter. + */ + public static Vector3f barycenter(ReadableMesh mesh) { + Vector3f barycenter = new Vector3f(0f, 0f, 0f); + double total = 0.0; + for (Mesh.Face face : mesh.getFaces()) { + double area = face.area(); + if (area > 0.0) { + barycenter.add(face.centroid().mul((float) area)); + total += area; + } } + barycenter.div((float) total); + return barycenter; + } - /** - * Calculates and returns the bounds for the provided mesh. - * - * @param mesh Mesh for which bounds should be calculated. - * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y, max_z, min_z} - */ - public static float[] bounds(ReadableMesh mesh) { - /* Extract all vertices that are part of a face. */ - List vertices = new ArrayList<>(mesh.numberOfVertices()); - for (Mesh.Face face : mesh.getFaces()) { - for (Mesh.Vertex vertex : face.getVertices()) { - vertices.add(vertex.getPosition()); - } - } - - return bounds(vertices); + /** + * Calculates and returns the bounds for the provided mesh. + * + * @param mesh Mesh for which bounds should be calculated. + * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y, max_z, min_z} + */ + public static float[] bounds(ReadableMesh mesh) { + /* Extract all vertices that are part of a face. */ + List vertices = new ArrayList<>(mesh.numberOfVertices()); + for (Mesh.Face face : mesh.getFaces()) { + for (Mesh.Vertex vertex : face.getVertices()) { + vertices.add(vertex.getPosition()); + } } - /** - * Calculates and returns the bounds for the provided mesh. - * - * @param vertices Vertices for which bounds should be calculated. - * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y, max_z, min_z} - */ - public static float[] bounds(List vertices) { - /* If no vertices are in the list, the box is zero. */ - if (vertices.isEmpty()) { - return new float[6]; - } + return bounds(vertices); + } - /* Initialize the bounding-box. */ - float bounds[] = { - -Float.MAX_VALUE, Float.MAX_VALUE, - -Float.MAX_VALUE, Float.MAX_VALUE, - -Float.MAX_VALUE, Float.MAX_VALUE - }; + /** + * Calculates and returns the bounds for the provided mesh. + * + * @param vertices Vertices for which bounds should be calculated. + * @return Float-array spanning the bounds: {max_x, min_x, max_y, min_y, max_z, min_z} + */ + public static float[] bounds(List vertices) { + /* If no vertices are in the list, the box is zero. */ + if (vertices.isEmpty()) { + return new float[6]; + } - /* Find max and min y-values. */ - for(Vector3fc vertex : vertices) { - if (vertex.x() > bounds[0]) { - bounds[0] = vertex.x(); - } - if (vertex.x() < bounds[1]) { - bounds[1] = vertex.x(); - } - if (vertex.y() > bounds[2]) { - bounds[2] = vertex.y(); - } - if (vertex.y() < bounds[3]) { - bounds[3] = vertex.y(); - } - if (vertex.z() > bounds[4]) { - bounds[4] = vertex.z(); - } - if (vertex.z() < bounds[5]) { - bounds[5] = vertex.z(); - } - } + /* Initialize the bounding-box. */ + float bounds[] = { + -Float.MAX_VALUE, Float.MAX_VALUE, + -Float.MAX_VALUE, Float.MAX_VALUE, + -Float.MAX_VALUE, Float.MAX_VALUE + }; - /* Return bounding-box. */ - return bounds; + /* Find max and min y-values. */ + for (Vector3fc vertex : vertices) { + if (vertex.x() > bounds[0]) { + bounds[0] = vertex.x(); + } + if (vertex.x() < bounds[1]) { + bounds[1] = vertex.x(); + } + if (vertex.y() > bounds[2]) { + bounds[2] = vertex.y(); + } + if (vertex.y() < bounds[3]) { + bounds[3] = vertex.y(); + } + if (vertex.z() > bounds[4]) { + bounds[4] = vertex.z(); + } + if (vertex.z() < bounds[5]) { + bounds[5] = vertex.z(); + } } + + /* Return bounding-box. */ + return bounds; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshTransformUtil.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshTransformUtil.java index e87da827b..10dd8e4cd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshTransformUtil.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/mesh/MeshTransformUtil.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.util.mesh; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; import org.ejml.data.DMatrixRMaj; import org.ejml.dense.row.factory.DecompositionFactory_DDRM; import org.ejml.interfaces.decomposition.EigenDecomposition_F64; @@ -11,208 +14,195 @@ import org.vitrivr.cineast.core.data.m3d.ReadableMesh; import org.vitrivr.cineast.core.data.m3d.WritableMesh; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - /** - * A collection of utilities surrounding transformation of Meshes. Includes methods to scale, move, center or apply the - * Karhunen–Loève (KHL) Transformation [1] for pose normalisation. - * - * [1] Vranic, D. V., Saupe, D., & Richter, J. (2001). Tools for 3D-object retrieval: Karhunen-Loeve transform and\nspherical harmonics. - * 2001 IEEE Fourth Workshop on Multimedia Signal Processing (Cat. No.01TH8564), 0–5. http://doi.org/10.1109/MMSP.2001.962749 - * + * A collection of utilities surrounding transformation of Meshes. Includes methods to scale, move, center or apply the Karhunen–Loève (KHL) Transformation [1] for pose normalisation. + *

    + * [1] Vranic, D. V., Saupe, D., & Richter, J. (2001). Tools for 3D-object retrieval: Karhunen-Loeve transform and\nspherical harmonics. 2001 IEEE Fourth Workshop on Multimedia Signal Processing (Cat. No.01TH8564), 0–5. http://doi.org/10.1109/MMSP.2001.962749 */ public final class MeshTransformUtil { - /** - * Private constructor; cannot be instantiated. - */ - private MeshTransformUtil() {} - - /** - * Performs a Karhunen–Loève (KHL) Transformation on the provided Mesh by executing the following steps - * according to [1]: - * - *

      - *
    1. Move the Mesh's barycenter to the origin.
      2. - *
    2. Rotate the Mesh's so that its PCA axis becomes aligned with the coordinate-system's axis.
      3. - *
    3. Scale the mesh (usually to unit-size).
      4. - *
    - * - * This transformation is done on a copy of the Mesh, which is returned. The original Mesh is not affected by the operation. - * - * The KHL Transform is a default tool for pose estimation. - * - * @param mesh Mesh that should be transformed. - * @param size Length of the longest edge of the mesh's bounding box (for scaling). - * @return KHL transformed copy of the original Mesh - */ - public static Mesh khlTransform(Mesh mesh, float size) { - Mesh copy = new Mesh(mesh); - khlTransformInPlace(copy, size); - return copy; - } - /** - * Performs a Karhunen–Loève (KHL) Transformation on the provided Mesh by executing the following steps - * according to [1]: - * - *
      - *
    1. Move the Mesh's barycenter to the origin.
      2. - *
    2. Rotate the Mesh's so that its PCA axis becomes aligned with the coordinate-system's axis.
      3. - *
    3. Scale the mesh (usually to unit-size).
      4. - *
    - * - * The KHL Transform is a default tool for pose estimation. - * - * Important: This transformation is done in place and affects the original Mesh. - * - * @param mesh Mesh that should be transformed. - * @param size Length of the longest edge of the mesh's bounding box (for scaling). - */ - public static void khlTransformInPlace(WritableMesh mesh, float size) { - /* Check if Mesh is empty. */ - if (mesh.isEmpty()) { - return; - } - - /* 1) Scale the mesh. */ - MeshTransformUtil.scaleInPlace(mesh, size); - - /* 2) Rotate the mesh along its PCA axis. */ - Vector3fc barycenter = mesh.barycenter(); - - /* Prepare an empty covariance matrix. */ - DMatrixRMaj covariance = new DMatrixRMaj(3,3); - - List faces = mesh.getFaces(); - long vertices = 0; - for (Mesh.Face face : faces) { - for (Mesh.Vertex v : face.getVertices()) { - Vector3f vm = (new Vector3f(v.getPosition())).sub(barycenter); - covariance.add(0,0, vm.x * vm.x); - covariance.add(0,1, vm.y * vm.x); - covariance.add(0,2, vm.z * vm.x); - covariance.add(1,0, vm.y * vm.x); - covariance.add(1,1, vm.y * vm.y); - covariance.add(1,2, vm.y * vm.z); - covariance.add(2,0, vm.z * vm.x); - covariance.add(2,1, vm.z * vm.y); - covariance.add(2,2, vm.z * vm.z); - vertices++; - } - } - - /* Normalizes the matrix. */ - for (int i=0; i> eigenvectors = MeshTransformUtil.getEigenvectors(covariance); - - Matrix4f rotation = new Matrix4f(); - - /* 2b) Apply first rotation: Largest spread should occur along the x-axis. */ - Vector3f xaxis = new Vector3f(1.0f,0.0f,0.0f); - Vector3f yaxis = new Vector3f(0.0f,1.0f,0.0f); - float angleX = xaxis.angle(eigenvectors.get(2).second); - rotation.rotate(angleX, xaxis.cross(eigenvectors.get(2).second)); - - /* 2c) Apply second rotation: Second largest spread should occur along the y-axis. */ - float angleY = yaxis.angle(eigenvectors.get(1).second); - rotation.rotate(angleY, yaxis.cross(eigenvectors.get(1).second)); - mesh.transform(rotation); - - /* 3) Center the mesh. */ - MeshTransformUtil.centerInPlace(mesh); + /** + * Private constructor; cannot be instantiated. + */ + private MeshTransformUtil() { + } + + /** + * Performs a Karhunen–Loève (KHL) Transformation on the provided Mesh by executing the following steps according to [1]: + * + *
      + *
    1. Move the Mesh's barycenter to the origin.
      2. + *
    2. Rotate the Mesh's so that its PCA axis becomes aligned with the coordinate-system's axis.
      3. + *
    3. Scale the mesh (usually to unit-size).
      4. + *
    + *

    + * This transformation is done on a copy of the Mesh, which is returned. The original Mesh is not affected by the operation. + *

    + * The KHL Transform is a default tool for pose estimation. + * + * @param mesh Mesh that should be transformed. + * @param size Length of the longest edge of the mesh's bounding box (for scaling). + * @return KHL transformed copy of the original Mesh + */ + public static Mesh khlTransform(Mesh mesh, float size) { + Mesh copy = new Mesh(mesh); + khlTransformInPlace(copy, size); + return copy; + } + + /** + * Performs a Karhunen–Loève (KHL) Transformation on the provided Mesh by executing the following steps according to [1]: + * + *

      + *
    1. Move the Mesh's barycenter to the origin.
      2. + *
    2. Rotate the Mesh's so that its PCA axis becomes aligned with the coordinate-system's axis.
      3. + *
    3. Scale the mesh (usually to unit-size).
      4. + *
    + *

    + * The KHL Transform is a default tool for pose estimation. + * + * Important: This transformation is done in place and affects the original Mesh. + * + * @param mesh Mesh that should be transformed. + * @param size Length of the longest edge of the mesh's bounding box (for scaling). + */ + public static void khlTransformInPlace(WritableMesh mesh, float size) { + /* Check if Mesh is empty. */ + if (mesh.isEmpty()) { + return; } - /** - * Scales the provided Mesh so that the length of the largest edge of its bounding-box - * equals to the provided size. This transformation is done on a copy of the Mesh, which is - * returned. The original Mesh is not affected by the operation. - * - * @param mesh Mesh that should be scaled. - * @param size Length of the longest edge of the mesh's bounding box. - * @return Scaled copy of the original Mesh - */ - public static Mesh scale(ReadableMesh mesh, float size) { - Mesh copy = new Mesh(mesh); - scaleInPlace(copy, size); - return copy; + /* 1) Scale the mesh. */ + MeshTransformUtil.scaleInPlace(mesh, size); + + /* 2) Rotate the mesh along its PCA axis. */ + Vector3fc barycenter = mesh.barycenter(); + + /* Prepare an empty covariance matrix. */ + DMatrixRMaj covariance = new DMatrixRMaj(3, 3); + + List faces = mesh.getFaces(); + long vertices = 0; + for (Mesh.Face face : faces) { + for (Mesh.Vertex v : face.getVertices()) { + Vector3f vm = (new Vector3f(v.getPosition())).sub(barycenter); + covariance.add(0, 0, vm.x * vm.x); + covariance.add(0, 1, vm.y * vm.x); + covariance.add(0, 2, vm.z * vm.x); + covariance.add(1, 0, vm.y * vm.x); + covariance.add(1, 1, vm.y * vm.y); + covariance.add(1, 2, vm.y * vm.z); + covariance.add(2, 0, vm.z * vm.x); + covariance.add(2, 1, vm.z * vm.y); + covariance.add(2, 2, vm.z * vm.z); + vertices++; + } } - /** - * Scales the provided Mesh so that the length of the largest edge of its bounding-box - * equals to the provided size. - * - * Important: This transformation is done in place and affects the original Mesh. - * - * @param mesh Mesh that should be scaled. - * @param size Length of the longest edge of the mesh's bounding box. - */ - public static void scaleInPlace(WritableMesh mesh, float size) { - if (mesh.isEmpty()) { - return; - } - float[] bounds = MeshMathUtil.bounds(mesh); - float max = Math.max(bounds[0]-bounds[1], Math.max(bounds[2]-bounds[3], bounds[4]-bounds[5])); - mesh.scale(size/max); + /* Normalizes the matrix. */ + for (int i = 0; i < covariance.data.length; i++) { + covariance.data[i] /= vertices; } - /** - * Moves the provided Mesh so that its barycenter is positioned at the origin [0,0,0]. This - * transformation is done on a copy of the Mesh, which is returned. The original Mesh is not - * affected by the operation. - * - * @param mesh Mesh that should be centered. - * @return Centered copy of the original Mesh - */ - public static Mesh center(ReadableMesh mesh) { - Mesh copy = new Mesh(mesh); - centerInPlace(copy); - return copy; + /* 2a) List of eigenvectors sorted by eigenvalue in descending order. */ + List> eigenvectors = MeshTransformUtil.getEigenvectors(covariance); + + Matrix4f rotation = new Matrix4f(); + + /* 2b) Apply first rotation: Largest spread should occur along the x-axis. */ + Vector3f xaxis = new Vector3f(1.0f, 0.0f, 0.0f); + Vector3f yaxis = new Vector3f(0.0f, 1.0f, 0.0f); + float angleX = xaxis.angle(eigenvectors.get(2).second); + rotation.rotate(angleX, xaxis.cross(eigenvectors.get(2).second)); + + /* 2c) Apply second rotation: Second largest spread should occur along the y-axis. */ + float angleY = yaxis.angle(eigenvectors.get(1).second); + rotation.rotate(angleY, yaxis.cross(eigenvectors.get(1).second)); + mesh.transform(rotation); + + /* 3) Center the mesh. */ + MeshTransformUtil.centerInPlace(mesh); + } + + /** + * Scales the provided Mesh so that the length of the largest edge of its bounding-box equals to the provided size. This transformation is done on a copy of the Mesh, which is returned. The original Mesh is not affected by the operation. + * + * @param mesh Mesh that should be scaled. + * @param size Length of the longest edge of the mesh's bounding box. + * @return Scaled copy of the original Mesh + */ + public static Mesh scale(ReadableMesh mesh, float size) { + Mesh copy = new Mesh(mesh); + scaleInPlace(copy, size); + return copy; + } + + /** + * Scales the provided Mesh so that the length of the largest edge of its bounding-box equals to the provided size. + * + * Important: This transformation is done in place and affects the original Mesh. + * + * @param mesh Mesh that should be scaled. + * @param size Length of the longest edge of the mesh's bounding box. + */ + public static void scaleInPlace(WritableMesh mesh, float size) { + if (mesh.isEmpty()) { + return; } - - /** - * Moves the provided Mesh so that its barycenter is positioned at the origin [0,0,0]. - * - *Important: This transformation is done in place and affects the original Mesh. - * - * @param mesh Centered Mesh. - */ - public static void centerInPlace(WritableMesh mesh) { - if (mesh.isEmpty()) { - return; - } - Vector3f center = new Vector3f(mesh.barycenter()); - mesh.move(center.negate()); + float[] bounds = MeshMathUtil.bounds(mesh); + float max = Math.max(bounds[0] - bounds[1], Math.max(bounds[2] - bounds[3], bounds[4] - bounds[5])); + mesh.scale(size / max); + } + + /** + * Moves the provided Mesh so that its barycenter is positioned at the origin [0,0,0]. This transformation is done on a copy of the Mesh, which is returned. The original Mesh is not affected by the operation. + * + * @param mesh Mesh that should be centered. + * @return Centered copy of the original Mesh + */ + public static Mesh center(ReadableMesh mesh) { + Mesh copy = new Mesh(mesh); + centerInPlace(copy); + return copy; + } + + /** + * Moves the provided Mesh so that its barycenter is positioned at the origin [0,0,0]. + * + * Important: This transformation is done in place and affects the original Mesh. + * + * @param mesh Centered Mesh. + */ + public static void centerInPlace(WritableMesh mesh) { + if (mesh.isEmpty()) { + return; } - - /** - * Calculates a List of eigenvalues and eigenvectors from the provided 3x3 (covariance) matrix. - * - * @param matrix 3x3 Matrix to derive the eigenvalues and eigenvectors from. - * @return List of pairs containing both the eigenvalues and the eigenvectors. The entries are - * sorted in ascending order of the eigenvalue. - */ - private static List> getEigenvectors(DMatrixRMaj matrix) { - List> eigenvectors = new ArrayList<>(); - - EigenDecomposition_F64 eig = DecompositionFactory_DDRM.eig(3, true); - eig.decompose(matrix); - - int eigValNum = eig.getNumberOfEigenvalues(); - for(int i = 0; i < eigValNum; i++){ - DMatrixRMaj eigMat = eig.getEigenVector(i); - if(eigMat != null){ - eigenvectors.add(new Pair<>((float)eig.getEigenvalue(i).getReal(), new Vector3f((float)eigMat.get(0,0), (float)eigMat.get(1,0), (float)eigMat.get(2,0)))); - } - } - - eigenvectors.sort(Comparator.comparing(e -> e.first)); - - return eigenvectors; + Vector3f center = new Vector3f(mesh.barycenter()); + mesh.move(center.negate()); + } + + /** + * Calculates a List of eigenvalues and eigenvectors from the provided 3x3 (covariance) matrix. + * + * @param matrix 3x3 Matrix to derive the eigenvalues and eigenvectors from. + * @return List of pairs containing both the eigenvalues and the eigenvectors. The entries are sorted in ascending order of the eigenvalue. + */ + private static List> getEigenvectors(DMatrixRMaj matrix) { + List> eigenvectors = new ArrayList<>(); + + EigenDecomposition_F64 eig = DecompositionFactory_DDRM.eig(3, true); + eig.decompose(matrix); + + int eigValNum = eig.getNumberOfEigenvalues(); + for (int i = 0; i < eigValNum; i++) { + DMatrixRMaj eigMat = eig.getEigenVector(i); + if (eigMat != null) { + eigenvectors.add(new Pair<>((float) eig.getEigenvalue(i).getReal(), new Vector3f((float) eigMat.get(0, 0), (float) eigMat.get(1, 0), (float) eigMat.get(2, 0)))); + } } + + eigenvectors.sort(Comparator.comparing(e -> e.first)); + + return eigenvectors; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/AudioParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/AudioParser.java index b959600f5..d6b7b9a6a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/AudioParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/AudioParser.java @@ -1,89 +1,90 @@ package org.vitrivr.cineast.core.util.web; -import org.vitrivr.cineast.core.data.frames.AudioDescriptor; -import org.vitrivr.cineast.core.data.frames.AudioFrame; -import org.vitrivr.cineast.core.util.LogHelper; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.UnsupportedAudioFileException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; +import org.vitrivr.cineast.core.data.frames.AudioDescriptor; +import org.vitrivr.cineast.core.data.frames.AudioFrame; +import org.vitrivr.cineast.core.util.LogHelper; public class AudioParser extends DataURLParser { - /** Mimetype of supported data URL's. Only WAV audio is currently supported. */ - private static final String MIME_TYPE = "audio/wav"; - /** - * Converts a Base64 data URL into a List auf AudioFrames. - * - * @param dataUrl String containing the data url. - * @return List of AudioFrames or empty list, if conversion failed. - */ - public static List parseWaveAudio(String dataUrl, float samplerate, int channels) { + /** + * Mimetype of supported data URL's. Only WAV audio is currently supported. + */ + private static final String MIME_TYPE = "audio/wav"; - ArrayList list = new ArrayList<>(); + /** + * Converts a Base64 data URL into a List auf AudioFrames. + * + * @param dataUrl String containing the data url. + * @return List of AudioFrames or empty list, if conversion failed. + */ + public static List parseWaveAudio(String dataUrl, float samplerate, int channels) { - /* Convert Base64 string into byte array. */ - byte[] bytes = dataURLtoByteArray(dataUrl, MIME_TYPE); - if (bytes == null) { - return list; - } + ArrayList list = new ArrayList<>(); + /* Convert Base64 string into byte array. */ + byte[] bytes = dataURLtoByteArray(dataUrl, MIME_TYPE); + if (bytes == null) { + return list; + } - try { - /* Read data as AudioInputStream and re-sample it. */ - ByteArrayInputStream rawByteStream = new ByteArrayInputStream(bytes); - AudioInputStream inputAudio = AudioSystem.getAudioInputStream(rawByteStream); - AudioFormat targetFormat = new AudioFormat(samplerate, 16, channels, true, false); - AudioInputStream convertedAudio = AudioSystem.getAudioInputStream(targetFormat, inputAudio); + try { + /* Read data as AudioInputStream and re-sample it. */ + ByteArrayInputStream rawByteStream = new ByteArrayInputStream(bytes); + AudioInputStream inputAudio = AudioSystem.getAudioInputStream(rawByteStream); + AudioFormat targetFormat = new AudioFormat(samplerate, 16, channels, true, false); + AudioInputStream convertedAudio = AudioSystem.getAudioInputStream(targetFormat, inputAudio); - /* Estimate duration of the segment. */ - long duration = (long) Math.floor((convertedAudio.getFrameLength()/samplerate)); + /* Estimate duration of the segment. */ + long duration = (long) Math.floor((convertedAudio.getFrameLength() / samplerate)); - /* Constants: - * - Length of a single AudioFrame (in bytes) - * - Total length of the frames data in the AudioInputStream. - */ + /* Constants: + * - Length of a single AudioFrame (in bytes) + * - Total length of the frames data in the AudioInputStream. + */ - final int framesize = 2048; - final int bytesPerSample = targetFormat.getSampleSizeInBits()/8; - final int bytesPerFrame = framesize * bytesPerSample; - final long length = convertedAudio.getFrameLength() * bytesPerSample * targetFormat.getChannels(); + final int framesize = 2048; + final int bytesPerSample = targetFormat.getSampleSizeInBits() / 8; + final int bytesPerFrame = framesize * bytesPerSample; + final long length = convertedAudio.getFrameLength() * bytesPerSample * targetFormat.getChannels(); - /* - * Read the data into constant length AudioFrames. - */ - int read = 0; - int idx = 0; - long timestamp = 0; - boolean done = false; - while (!done) { - /* Allocate a byte-array for the frames-data. */ - byte[] data = null; - if (read + bytesPerFrame < length) { - data = new byte[bytesPerFrame]; - } else { - data = new byte[(int)(length-read)]; - done = true; - } - /* Read frames-data and create AudioFrame. */ - int len = convertedAudio.read(data, 0, data.length); - list.add(new AudioFrame(idx, timestamp, data, new AudioDescriptor(targetFormat.getSampleRate(), targetFormat.getChannels(), duration))); - timestamp += (len*1000.0f)/(bytesPerSample * targetFormat.getChannels() * targetFormat.getSampleRate()); - idx += 1; - read += len; - } - } catch (UnsupportedAudioFileException e) { - LOGGER.error("Could not create audio frames from Base64 input because the file-format is not supported. {}", LogHelper.getStackTrace(e)); - } catch (IOException e) { - LOGGER.error("Could not create audio frames from Base64 input due to a serious IO error: {}", LogHelper.getStackTrace(e)); + /* + * Read the data into constant length AudioFrames. + */ + int read = 0; + int idx = 0; + long timestamp = 0; + boolean done = false; + while (!done) { + /* Allocate a byte-array for the frames-data. */ + byte[] data = null; + if (read + bytesPerFrame < length) { + data = new byte[bytesPerFrame]; + } else { + data = new byte[(int) (length - read)]; + done = true; } - - return list; + /* Read frames-data and create AudioFrame. */ + int len = convertedAudio.read(data, 0, data.length); + list.add(new AudioFrame(idx, timestamp, data, new AudioDescriptor(targetFormat.getSampleRate(), targetFormat.getChannels(), duration))); + timestamp += (len * 1000.0f) / (bytesPerSample * targetFormat.getChannels() * targetFormat.getSampleRate()); + idx += 1; + read += len; + } + } catch (UnsupportedAudioFileException e) { + LOGGER.error("Could not create audio frames from Base64 input because the file-format is not supported. {}", LogHelper.getStackTrace(e)); + } catch (IOException e) { + LOGGER.error("Could not create audio frames from Base64 input due to a serious IO error: {}", LogHelper.getStackTrace(e)); } + + return list; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/DataURLParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/DataURLParser.java index 53a706f36..766873f5b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/DataURLParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/DataURLParser.java @@ -2,137 +2,135 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.codec.binary.Base64; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Optional; +import org.apache.commons.codec.binary.Base64; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class DataURLParser { - protected static final Logger LOGGER = LogManager.getLogger(); - - /** MimeType for JSON data. */ - private static final String JSON_MIME_TYPE = "application/json"; - - - protected DataURLParser() {} - - /** - * Tries to convert a Base64 data URL to a JsonNode and returns that - * JsonNode. - * - * @param dataUrl String containing the data url. - * @return Optional JsonNode - */ - public static Optional dataURLtoJsonNode(String dataUrl) { - /* Convert Base64 string into byte array. */ - byte[] bytes = dataURLtoByteArray(dataUrl, JSON_MIME_TYPE); - ObjectMapper mapper = new ObjectMapper(); - try { - return Optional.of(mapper.readTree(bytes)); - } catch (IOException e) { - LOGGER.error("Exception occurred while parsing data URL to JSON: {}", e); - return Optional.empty(); - } + protected static final Logger LOGGER = LogManager.getLogger(); + + /** + * MimeType for JSON data. + */ + private static final String JSON_MIME_TYPE = "application/json"; + + + protected DataURLParser() { + } + + /** + * Tries to convert a Base64 data URL to a JsonNode and returns that JsonNode. + * + * @param dataUrl String containing the data url. + * @return Optional JsonNode + */ + public static Optional dataURLtoJsonNode(String dataUrl) { + /* Convert Base64 string into byte array. */ + byte[] bytes = dataURLtoByteArray(dataUrl, JSON_MIME_TYPE); + ObjectMapper mapper = new ObjectMapper(); + try { + return Optional.of(mapper.readTree(bytes)); + } catch (IOException e) { + LOGGER.error("Exception occurred while parsing data URL to JSON: {}", e); + return Optional.empty(); + } + } + + + /** + * Converts a Base64 data URL to a byte array and returns it. + * + * @param dataUrl String containing the data url. + * @return Byte array of the data. + */ + public static byte[] dataURLtoByteArray(String dataUrl) { + dataUrl = dataUrl.replace(' ', '+'); + + /* Check if string is actually a valid data URL. */ + if (!dataUrl.startsWith("data:")) { + LOGGER.warn("This is not a valid data URL."); + return null; + } + + /* Convert and return byte array. */ + int headerLength = dataUrl.indexOf(','); + String base46data = dataUrl.substring(headerLength + 1); + return Base64.decodeBase64(base46data); + } + + /** + * Converts a Base64 data URL to a byte array and returns it. Only URLs that contain the verify substring at position five of the String will be converted! + * + * @param dataUrl String containing the data url. + * @param verify Substring that must be contained at position 5 in order for the data to be converted. + * @return Byte array of the data. + */ + public static byte[] dataURLtoByteArray(String dataUrl, String verify) { + if (dataUrl == null) { + return null; } + dataUrl = dataUrl.replace(' ', '+'); + /* Check data URL. */ + if (!isValidDataUrl(dataUrl, verify)) { + return null; + } - /** - * Converts a Base64 data URL to a byte array and returns it. - * - * @param dataUrl String containing the data url. - * @return Byte array of the data. - */ - public static byte[] dataURLtoByteArray(String dataUrl) { - dataUrl = dataUrl.replace(' ', '+'); - - /* Check if string is actually a valid data URL. */ - if (!dataUrl.startsWith("data:")) { - LOGGER.warn("This is not a valid data URL."); - return null; - } - - /* Convert and return byte array. */ - int headerLength = dataUrl.indexOf(','); - String base46data = dataUrl.substring(headerLength + 1); - return Base64.decodeBase64(base46data); + /* Convert and return byte array. */ + int headerLength = dataUrl.indexOf(','); + String base46data = dataUrl.substring(headerLength + 1); + return Base64.decodeBase64(base46data); + } + + /** + * Converts a Base64 data URL to a UTF8 String and returns it. Only URLs that contain the verify substring at position five of the String will be converted! + * + * @param dataUrl String containing the data url. + * @param verify Substring that must be contained at position 5 in order for the data to be converted. + * @return String representation of the data. + */ + public static String dataURLtoString(String dataUrl, String verify) { + if (dataUrl == null) { + return null; } + dataUrl = dataUrl.replace(' ', '+'); - /** - * Converts a Base64 data URL to a byte array and returns it. Only URLs that contain the - * verify substring at position five of the String will be converted! - * - * @param dataUrl String containing the data url. - * @param verify Substring that must be contained at position 5 in order for the data to be converted. - * @return Byte array of the data. - */ - public static byte[] dataURLtoByteArray(String dataUrl, String verify) { - if (dataUrl == null) { - return null; - } - dataUrl = dataUrl.replace(' ', '+'); - - /* Check data URL. */ - if (!isValidDataUrl(dataUrl, verify)) { - return null; - } - - /* Convert and return byte array. */ - int headerLength = dataUrl.indexOf(','); - String base46data = dataUrl.substring(headerLength + 1); - return Base64.decodeBase64(base46data); + /* Check data URL. */ + if (!isValidDataUrl(dataUrl, verify)) { + return null; } - /** - * Converts a Base64 data URL to a UTF8 String and returns it. Only URLs that contain the verify substring at position five of the - * String will be converted! - * - * @param dataUrl String containing the data url. - * @param verify Substring that must be contained at position 5 in order for the data to be converted. - * @return String representation of the data. - */ - public static String dataURLtoString(String dataUrl, String verify) { - if (dataUrl == null) { - return null; - } - dataUrl = dataUrl.replace(' ', '+'); - - /* Check data URL. */ - if (!isValidDataUrl(dataUrl, verify)) { - return null; - } - - /* Convert and return byte array. */ - int headerLength = dataUrl.indexOf(','); - String base46data = dataUrl.substring(headerLength + 1); - return new String (Base64.decodeBase64(base46data), StandardCharsets.UTF_8); + /* Convert and return byte array. */ + int headerLength = dataUrl.indexOf(','); + String base46data = dataUrl.substring(headerLength + 1); + return new String(Base64.decodeBase64(base46data), StandardCharsets.UTF_8); + } + + /** + * Checks if the provided data URL is actually a valid data URL. Returns true, if so and false otherwise. + * + * @param dataUrl Data URL that should be checked. + * @param verify Substring that must be contained at position 5 in order for the data to be valid. + * @return True if valid and false otherwise. + */ + public static boolean isValidDataUrl(String dataUrl, String verify) { + /* Check if string is actually a valid data URL. */ + if (!dataUrl.startsWith("data:")) { + LOGGER.warn("{} is not a valid data URL.", dataUrl); + return false; } - /** - * Checks if the provided data URL is actually a valid data URL. Returns true, if so and false - * otherwise. - * - * @param dataUrl Data URL that should be checked. - * @param verify Substring that must be contained at position 5 in order for the data to be valid. - * @return True if valid and false otherwise. - */ - public static boolean isValidDataUrl(String dataUrl, String verify) { - /* Check if string is actually a valid data URL. */ - if (!dataUrl.startsWith("data:")) { - LOGGER.warn("{} is not a valid data URL.", dataUrl); - return false; - } - - /* Check if data URL is of supported type. */ - if (!dataUrl.substring(5, 5 + verify.length()).equals(verify)) { - LOGGER.warn("Data URL does not have a supported type."); - return false; - } - - return true; + /* Check if data URL is of supported type. */ + if (!dataUrl.substring(5, 5 + verify.length()).equals(verify)) { + LOGGER.warn("Data URL does not have a supported type."); + return false; } + + return true; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java index aa3ea2d57..345bd3794 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java @@ -1,63 +1,62 @@ package org.vitrivr.cineast.core.util.web; -import org.apache.commons.codec.binary.Base64; -import org.vitrivr.cineast.core.util.LogHelper; - -import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import javax.imageio.ImageIO; +import org.apache.commons.codec.binary.Base64; +import org.vitrivr.cineast.core.util.LogHelper; public class ImageParser extends DataURLParser { - /** - * Converts a Base64 data URL to a BufferedImage. - * - * @param dataUrl String containing the data url. - * @return BufferedImage or null, if conversion failed. - */ - public static BufferedImage dataURLtoBufferedImage(String dataUrl) { - - /* Convert Base64 string into byte array. */ - byte[] bytes = dataURLtoByteArray(dataUrl, "image/"); - if (bytes == null) { - return null; - } - - ByteArrayInputStream binstream = new ByteArrayInputStream(bytes); - - BufferedImage bimg; - try { - bimg = ImageIO.read(binstream); - } catch (IOException e) { - LOGGER.error("could not make image, {}", LogHelper.getStackTrace(e)); - return null; - } - - return bimg; + + /** + * Converts a Base64 data URL to a BufferedImage. + * + * @param dataUrl String containing the data url. + * @return BufferedImage or null, if conversion failed. + */ + public static BufferedImage dataURLtoBufferedImage(String dataUrl) { + + /* Convert Base64 string into byte array. */ + byte[] bytes = dataURLtoByteArray(dataUrl, "image/"); + if (bytes == null) { + return null; } - public static String BufferedImageToDataURL(BufferedImage img, String format){ - ByteArrayOutputStream bouts = new ByteArrayOutputStream(); - try { - ImageIO.write(img, format, bouts); - } catch (IOException e) { - LOGGER.error("could not make image, {}", LogHelper.getStackTrace(e)); - return null; - } - String base64 = new String(Base64.encodeBase64(bouts.toByteArray())); - return "data:image/" + format + ";base64," + base64; + ByteArrayInputStream binstream = new ByteArrayInputStream(bytes); + + BufferedImage bimg; + try { + bimg = ImageIO.read(binstream); + } catch (IOException e) { + LOGGER.error("could not make image, {}", LogHelper.getStackTrace(e)); + return null; } - /** - * Checks, if provided data URL is a valid image. Returns true if so and false otherwise. - * No structural analysis is performed! Only the raw, data URL is being checked. - * - * @param dataUrl Data URL that should be checked. - * @return True, if data URL is a valid Three v4 JSON geometry. - */ - public static boolean isValidImage(String dataUrl) { - return isValidDataUrl(dataUrl, "image/"); + return bimg; + } + + public static String BufferedImageToDataURL(BufferedImage img, String format) { + ByteArrayOutputStream bouts = new ByteArrayOutputStream(); + try { + ImageIO.write(img, format, bouts); + } catch (IOException e) { + LOGGER.error("could not make image, {}", LogHelper.getStackTrace(e)); + return null; } + String base64 = new String(Base64.encodeBase64(bouts.toByteArray())); + return "data:image/" + format + ";base64," + base64; + } + + /** + * Checks, if provided data URL is a valid image. Returns true if so and false otherwise. No structural analysis is performed! Only the raw, data URL is being checked. + * + * @param dataUrl Data URL that should be checked. + * @return True, if data URL is a valid Three v4 JSON geometry. + */ + public static boolean isValidImage(String dataUrl) { + return isValidDataUrl(dataUrl, "image/"); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MeshParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MeshParser.java index 0f722a2ed..e68a88134 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MeshParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MeshParser.java @@ -3,82 +3,80 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import gnu.trove.map.hash.TObjectIntHashMap; +import java.io.IOException; import org.joml.Vector3f; import org.joml.Vector3i; import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.util.LogHelper; -import java.io.IOException; - public class MeshParser extends DataURLParser { - /** Some format specific constants. */ - private static final String MIME_TYPE = "application/3d-json"; - private static final String VERTICES_PROPERTY_NAME_THREEV4 = "vertices"; - private static final String NORMAL_PROPERTY_NAME_THREEV4 = "normals"; - private static final String ARRAY_PROPERTY_NAME_THREEV4 = "array"; + /** + * Some format specific constants. + */ + private static final String MIME_TYPE = "application/3d-json"; + private static final String VERTICES_PROPERTY_NAME_THREEV4 = "vertices"; + private static final String NORMAL_PROPERTY_NAME_THREEV4 = "normals"; + private static final String ARRAY_PROPERTY_NAME_THREEV4 = "array"; - /** - * Parses a Base64 encoded data url and treats it as Geometry JSON used by the Three.js JavaScript library. - * Tries to parse the structure into a 3D mesh. - * - * @param dataUrl Data URL that should be parsed. - * @return Mesh, if parsing fails that Mesh will be empty! - */ - public static Mesh parseThreeJSV4Geometry(String dataUrl) { - /* Convert Base64 string into byte array. */ - byte[] bytes = dataURLtoByteArray(dataUrl, MIME_TYPE); + /** + * Parses a Base64 encoded data url and treats it as Geometry JSON used by the Three.js JavaScript library. Tries to parse the structure into a 3D mesh. + * + * @param dataUrl Data URL that should be parsed. + * @return Mesh, if parsing fails that Mesh will be empty! + */ + public static Mesh parseThreeJSV4Geometry(String dataUrl) { + /* Convert Base64 string into byte array. */ + byte[] bytes = dataURLtoByteArray(dataUrl, MIME_TYPE); - ObjectMapper mapper = new ObjectMapper(); - try { - /* Read the JSON structure of the transmitted mesh data. */ - JsonNode node = mapper.readTree(bytes); - JsonNode vertices = node.get(VERTICES_PROPERTY_NAME_THREEV4); - if (vertices == null || !vertices.isArray() || vertices.size() == 0) { - LOGGER.error("Submitted mesh does not contain any vertices. Aborting..."); - return Mesh.EMPTY; - } + ObjectMapper mapper = new ObjectMapper(); + try { + /* Read the JSON structure of the transmitted mesh data. */ + JsonNode node = mapper.readTree(bytes); + JsonNode vertices = node.get(VERTICES_PROPERTY_NAME_THREEV4); + if (vertices == null || !vertices.isArray() || vertices.size() == 0) { + LOGGER.error("Submitted mesh does not contain any vertices. Aborting..."); + return Mesh.EMPTY; + } - /* Create new Mesh. */ - Mesh mesh = new Mesh(vertices.size()/9, vertices.size()/3); + /* Create new Mesh. */ + Mesh mesh = new Mesh(vertices.size() / 9, vertices.size() / 3); - /* Prepare helper structures. */ - TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); - int index = 0; - int[] vertexindices = new int[3]; + /* Prepare helper structures. */ + TObjectIntHashMap vertexBuffer = new TObjectIntHashMap<>(); + int index = 0; + int[] vertexindices = new int[3]; - /* Add all the vertices and normals in the structure. */ - for (int i=0; i<=vertices.size()-9; i+=9) { - for (int j=0; j<3; j++) { - int idx = i + 3*j; - Vector3f vertex = new Vector3f((float)vertices.get(idx).asDouble(), (float)vertices.get(idx+1).asDouble(),(float)vertices.get(idx+2).asDouble()); - if (!vertexBuffer.containsKey(vertex)) { - vertexBuffer.put(vertex, index++); - mesh.addVertex(vertex); - } - vertexindices[j] = vertexBuffer.get(vertex); - } + /* Add all the vertices and normals in the structure. */ + for (int i = 0; i <= vertices.size() - 9; i += 9) { + for (int j = 0; j < 3; j++) { + int idx = i + 3 * j; + Vector3f vertex = new Vector3f((float) vertices.get(idx).asDouble(), (float) vertices.get(idx + 1).asDouble(), (float) vertices.get(idx + 2).asDouble()); + if (!vertexBuffer.containsKey(vertex)) { + vertexBuffer.put(vertex, index++); + mesh.addVertex(vertex); + } + vertexindices[j] = vertexBuffer.get(vertex); + } - mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2])); - } + mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2])); + } - return mesh; - } catch (IOException e) { - LOGGER.error("Could not create 3d mesh from Base64 input because the file-format is not supported. {}", LogHelper.getStackTrace(e)); - return Mesh.EMPTY; - } + return mesh; + } catch (IOException e) { + LOGGER.error("Could not create 3d mesh from Base64 input because the file-format is not supported. {}", LogHelper.getStackTrace(e)); + return Mesh.EMPTY; } + } - /** - * Checks, if provided data URL is a valid Three v4 JSON geometry. Returns true if so and - * false otherwise. No structural analysis is performed! Only the raw, data URL is - * being checked. - * - * @param dataUrl Data URL that should be checked. - * @return True, if data URL is a valid Three v4 JSON geometry. - */ - public static boolean isValidThreeJSV4Geometry(String dataUrl) { - return isValidDataUrl(dataUrl, MIME_TYPE); - } + /** + * Checks, if provided data URL is a valid Three v4 JSON geometry. Returns true if so and false otherwise. No structural analysis is performed! Only the raw, data URL is being checked. + * + * @param dataUrl Data URL that should be checked. + * @return True, if data URL is a valid Three v4 JSON geometry. + */ + public static boolean isValidThreeJSV4Geometry(String dataUrl) { + return isValidDataUrl(dataUrl, MIME_TYPE); + } } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/data/LocationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/data/LocationTest.java index f1c36b906..13d72f84a 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/data/LocationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/data/LocationTest.java @@ -1,15 +1,15 @@ package org.vitrivr.cineast.core.data; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.ImmutableList; +import java.util.List; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class LocationTest { + private static final float COORDINATES_DELTA = 1e-5f; @Test @@ -49,7 +49,7 @@ public void testLongitudeWrapping() { @DisplayName("Invalid Float Array") public void testInvalidFloatArray() { List invalidArrays = ImmutableList - .of(new float[] {}, new float[]{ 0f }, new float[]{ 0f, 1f, 2f }); + .of(new float[]{}, new float[]{0f}, new float[]{0f, 1f, 2f}); for (float[] array : invalidArrays) { assertThrows(IllegalArgumentException.class, () -> Location.of(array)); } @@ -77,7 +77,7 @@ private static void assertNormalizedCoordinates(float expectedLat, float expecte private static List getTestLocations(float latitude, float longitude) { return ImmutableList.of( Location.of(latitude, longitude), - Location.of(new float[] { latitude, longitude }) + Location.of(new float[]{latitude, longitude}) ); } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 0ac640fc5..769f269a3 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -63,7 +63,7 @@ public abstract class DBIntegrationTest { private IntegrationDBProvider provider; @BeforeAll - void checkConnection(){ + void checkConnection() { provider = provider(); selector = provider.getSelector(); LOGGER.info("Trying to establish connection to Database"); @@ -240,29 +240,29 @@ void knnSearch() { @Disabled /* TODO: Currently not supported in Cottontail DB v0.12.0. Re-activate, once support is back. */ @DisplayName("Batched KNN search") void batchedKnnSearch() { - selector.open(testVectorTableName); - List queries = new ArrayList<>(); - queries.add(new float[]{0.001f, 1, 0}); - queries.add(new float[]{3.1f, 1, 0}); - queries.add(new float[]{4.8f, 1, 0}); - queryConfig.setDistanceIfEmpty(Distance.manhattan); - List configs = queries.stream().map(el -> new ReadableQueryConfig(queryConfig)).collect(Collectors.toList()); - List result = selector.getBatchedNearestNeighbours(1, queries, FEATURE_VECTOR_COL_NAME, SegmentDistanceElement.class, configs); - Assertions.assertEquals(3, result.size()); - Assertions.assertEquals("0", result.get(0).getSegmentId()); - Assertions.assertEquals(0.001, result.get(0).getDistance(), 0.0001); - Assertions.assertEquals("3", result.get(1).getSegmentId()); - Assertions.assertEquals(0.1, result.get(1).getDistance(), 0.0001); - Assertions.assertEquals("5", result.get(2).getSegmentId()); - Assertions.assertEquals(0.2, result.get(2).getDistance(), 0.0001); + selector.open(testVectorTableName); + List queries = new ArrayList<>(); + queries.add(new float[]{0.001f, 1, 0}); + queries.add(new float[]{3.1f, 1, 0}); + queries.add(new float[]{4.8f, 1, 0}); + queryConfig.setDistanceIfEmpty(Distance.manhattan); + List configs = queries.stream().map(el -> new ReadableQueryConfig(queryConfig)).collect(Collectors.toList()); + List result = selector.getBatchedNearestNeighbours(1, queries, FEATURE_VECTOR_COL_NAME, SegmentDistanceElement.class, configs); + Assertions.assertEquals(3, result.size()); + Assertions.assertEquals("0", result.get(0).getSegmentId()); + Assertions.assertEquals(0.001, result.get(0).getDistance(), 0.0001); + Assertions.assertEquals("3", result.get(1).getSegmentId()); + Assertions.assertEquals(0.1, result.get(1).getDistance(), 0.0001); + Assertions.assertEquals("5", result.get(2).getSegmentId()); + Assertions.assertEquals(0.2, result.get(2).getDistance(), 0.0001); } /** * Verify that a resultSet satisfies a certain condition for a given column * - * @param results the full resultset - * @param col column for which the provided function should be executed + * @param results the full resultset + * @param col column for which the provided function should be executed * @param function function to be executed on the {@link PrimitiveTypeProvider} value at the column */ private void checkContains(List> results, String col, Function function) { diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java index e6e3bd1ac..196bc2656 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.core.db.cottontaildb; import org.junit.jupiter.api.Test; -import org.vitrivr.cineast.core.db.DBIntegrationTest; import org.vitrivr.cineast.core.db.IntegrationDBProvider; import org.vitrivr.cineast.core.db.dao.MetadataTest; import org.vitrivr.cottontail.client.language.ddl.OptimizeEntity; diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java index 306f47269..6c06d4409 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java @@ -282,7 +282,7 @@ void findByKeysAndDomain() { for (int key : keys) { objSpec.add(new MetadataAccessSpecification(MetadataType.OBJECT, "one", String.valueOf(key))); segSpec.add(new MetadataAccessSpecification(MetadataType.SEGMENT, "one", String.valueOf(key))); - for (int i = 0; i < ELEMENT_COUNT/2; i++) { + for (int i = 0; i < ELEMENT_COUNT / 2; i++) { objDescriptors.add(objMetadataForID(i).get(key)); segDescriptors.add(segMetadataForID(i).get(key)); } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/TemporalTestCases.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/TemporalTestCases.java index ad7a6874d..556446ca5 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/TemporalTestCases.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/TemporalTestCases.java @@ -133,7 +133,7 @@ public void buildTestCase3() { segments1.add(descriptor1_5.getSegmentId()); segments1.add(descriptor1_7.getSegmentId()); /* optimal result is 1: (1,1), 2: (2,1) 3: (3, 0.5) , because 4 is not a valid result */ - TemporalObject t1 = new TemporalObject(segments1, descriptor1_1.getObjectId(), 2.5/3f); + TemporalObject t1 = new TemporalObject(segments1, descriptor1_1.getObjectId(), 2.5 / 3f); expectedResults.add(t1); this.segmentMap = segmentMap; diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/sequential/SequentialTemporalScoringAlgorithmTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/sequential/SequentialTemporalScoringAlgorithmTest.java index 15262da35..d7d1664bb 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/sequential/SequentialTemporalScoringAlgorithmTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/sequential/SequentialTemporalScoringAlgorithmTest.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import org.junit.jupiter.api.DisplayName; diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/timedistance/TimeDistanceTemporalScoringAlgorithmTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/timedistance/TimeDistanceTemporalScoringAlgorithmTest.java index bf3a9f82a..5e15ee14a 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/timedistance/TimeDistanceTemporalScoringAlgorithmTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/temporal/timedistance/TimeDistanceTemporalScoringAlgorithmTest.java @@ -12,6 +12,7 @@ import org.vitrivr.cineast.core.temporal.TemporalTestCases; public class TimeDistanceTemporalScoringAlgorithmTest { + @Test @DisplayName("Sanity temporal scoring") public void testSanity() { diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java index 370365367..091ed5eb4 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/dsp/fft/WindowFunctionTest.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.core.util.dsp.fft; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.vitrivr.cineast.core.util.dsp.fft.windows.BlackmanHarrisWindow; @@ -7,91 +10,88 @@ import org.vitrivr.cineast.core.util.dsp.fft.windows.RectangularWindow; import org.vitrivr.cineast.core.util.dsp.fft.windows.WindowFunction; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class WindowFunctionTest { - @Test - @DisplayName("Rectangular Window Test") - void testRectangularWindow() { - RectangularWindow window = new RectangularWindow(); - this.executeTest(window, 512); - this.executeTest(window, 1024); - this.executeTest(window, 2048); - this.executeTest(window, 4096); - } - @Test - @DisplayName("Hanning Window Test") - void testHanningWindow() { - HanningWindow window = new HanningWindow(); - this.executeTest(window, 512); - this.executeTest(window, 1024); - this.executeTest(window, 2048); - this.executeTest(window, 4096); - } + @Test + @DisplayName("Rectangular Window Test") + void testRectangularWindow() { + RectangularWindow window = new RectangularWindow(); + this.executeTest(window, 512); + this.executeTest(window, 1024); + this.executeTest(window, 2048); + this.executeTest(window, 4096); + } - @Test - @DisplayName("Blackman Harris Window Test") - void testBlackmanHarrisWindow() { - BlackmanHarrisWindow window = new BlackmanHarrisWindow(); - this.executeTest(window, 512); - this.executeTest(window, 1024); - this.executeTest(window, 2048); - this.executeTest(window, 4096); - } + @Test + @DisplayName("Hanning Window Test") + void testHanningWindow() { + HanningWindow window = new HanningWindow(); + this.executeTest(window, 512); + this.executeTest(window, 1024); + this.executeTest(window, 2048); + this.executeTest(window, 4096); + } + @Test + @DisplayName("Blackman Harris Window Test") + void testBlackmanHarrisWindow() { + BlackmanHarrisWindow window = new BlackmanHarrisWindow(); + this.executeTest(window, 512); + this.executeTest(window, 1024); + this.executeTest(window, 2048); + this.executeTest(window, 4096); + } - private void executeTest(WindowFunction function, int length) { - assertAll("Window Function Test (" + length +")", - () -> testWindowFunctionZeroOutside(function, length), - () -> testWindowFunctionSymmetry(function, length), - () -> testNormalization(function, length) - ); - } - /** - * Tests if a given WindowFunction satisfies the condition that it must be zero outside the - * definition of the window interval (i.e. for i < 0 or i > length). - * - * @param function WindowFunction to test - * @param length Length of the window. - * @return True if WindowFunction meets normalization criterion, false otherwise. - */ - private boolean testWindowFunctionZeroOutside(WindowFunction function, int length) { - for (int i=-length;i<0;i++) { - assertEquals(0.0, function.value(i, length), "The function '" + function.getClass().getSimpleName() + "' does not become zero at position i=" + i); - } - for (int i=length+1;i<=2*length;i++) { - assertEquals(0.0, function.value(i, length), "The function '" + function.getClass().getSimpleName() + "' does not become zero at position i=" + i); - } - return true; + private void executeTest(WindowFunction function, int length) { + assertAll("Window Function Test (" + length + ")", + () -> testWindowFunctionZeroOutside(function, length), + () -> testWindowFunctionSymmetry(function, length), + () -> testNormalization(function, length) + ); + } + + /** + * Tests if a given WindowFunction satisfies the condition that it must be zero outside the definition of the window interval (i.e. for i < 0 or i > length). + * + * @param function WindowFunction to test + * @param length Length of the window. + * @return True if WindowFunction meets normalization criterion, false otherwise. + */ + private boolean testWindowFunctionZeroOutside(WindowFunction function, int length) { + for (int i = -length; i < 0; i++) { + assertEquals(0.0, function.value(i, length), "The function '" + function.getClass().getSimpleName() + "' does not become zero at position i=" + i); + } + for (int i = length + 1; i <= 2 * length; i++) { + assertEquals(0.0, function.value(i, length), "The function '" + function.getClass().getSimpleName() + "' does not become zero at position i=" + i); } + return true; + } - /** - * Tests if a given WindowFunction satisfies the symmetry condition. - * - * @param function WindowFunction to test - * @param length Length of the window. - */ - private void testWindowFunctionSymmetry(WindowFunction function, int length) { - for (int i=0;i l || l < 0 || m < 0) { - assertThrows(IllegalArgumentException.class, () -> { - new AssociatedLegendrePolynomial(l_u, m_u); - }); - } else { - new AssociatedLegendrePolynomial(l_u, m_u); - } - } + /** + * Tests definition of AssociatedLegendrePolynomials, which are only defined for 0 <= m <= l + */ + @Test + @DisplayName("Test Definition") + public void testDefinition() { + for (int l = -10; l < 10; l++) { + for (int m = 0; m <= 2 * l; m++) { + final int l_u = l; + final int m_u = m; + if (m > l || l < 0 || m < 0) { + assertThrows(IllegalArgumentException.class, () -> { + new AssociatedLegendrePolynomial(l_u, m_u); + }); + } else { + new AssociatedLegendrePolynomial(l_u, m_u); } + } } - - /** - * Tests if orthogonality relation regarding a fixed l holds true for all l between 0 and 5. - */ - @Test - @DisplayName("Test Orthogonality (l)") - public void testOrthogonalityL() { - final double dx = 1e-4; - for (int l1=0;l1<=5;l1++) { - for (int l2=0;l2<=5;l2++) { - for (int m=0;m<=l1 && m<=l2;m++) { - final AssociatedLegendrePolynomial alp1 = new AssociatedLegendrePolynomial(l1,m); - final AssociatedLegendrePolynomial alp2 = new AssociatedLegendrePolynomial(l2,m); - double result = 0.0; - final double expected = (2.0)/(2*l1+1) * ((double)CombinatoricsUtils.factorial(l1+m)/(double)CombinatoricsUtils.factorial(l1-m)) * MathHelper.kronecker(l1,l2); - for (double x = -1.0; x <= 1.0; x+=dx) { - result += (alp1.value(x) * alp2.value(x)) * dx; - } - assertEquals(expected, result, 1e-3); - } - } - + } + + /** + * Tests if orthogonality relation regarding a fixed l holds true for all l between 0 and 5. + */ + @Test + @DisplayName("Test Orthogonality (l)") + public void testOrthogonalityL() { + final double dx = 1e-4; + for (int l1 = 0; l1 <= 5; l1++) { + for (int l2 = 0; l2 <= 5; l2++) { + for (int m = 0; m <= l1 && m <= l2; m++) { + final AssociatedLegendrePolynomial alp1 = new AssociatedLegendrePolynomial(l1, m); + final AssociatedLegendrePolynomial alp2 = new AssociatedLegendrePolynomial(l2, m); + double result = 0.0; + final double expected = (2.0) / (2 * l1 + 1) * ((double) CombinatoricsUtils.factorial(l1 + m) / (double) CombinatoricsUtils.factorial(l1 - m)) * MathHelper.kronecker(l1, l2); + for (double x = -1.0; x <= 1.0; x += dx) { + result += (alp1.value(x) * alp2.value(x)) * dx; + } + assertEquals(expected, result, 1e-3); } - } + } - /** - * Test P_00 (l=0, m=0) - */ - @Test - @DisplayName("Test P00") - public void testPOO() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(0,0); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(1.0, alp.value(x), 1e-8); - } } - - /** - * Test P_10 (l=1, m=0) - */ - @Test - @DisplayName("Test P10") - public void testP1O() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(1,0); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(x, alp.value(x), 1e-8); - } + } + + /** + * Test P_00 (l=0, m=0) + */ + @Test + @DisplayName("Test P00") + public void testPOO() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(0, 0); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(1.0, alp.value(x), 1e-8); } - - /** - * Test P_11 (l=1, m=1) - */ - @Test - @DisplayName("Test P11") - public void testP11() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(1,1); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(-Math.sqrt(1-Math.pow(x,2)), alp.value(x), 1e-8); - } + } + + /** + * Test P_10 (l=1, m=0) + */ + @Test + @DisplayName("Test P10") + public void testP1O() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(1, 0); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(x, alp.value(x), 1e-8); } - - /** - * Test P_20 (l=2, m=0) - */ - @Test - @DisplayName("Test P20") - public void testP20() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2,0); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(0.5 * (3*Math.pow(x,2)-1), alp.value(x), 1e-8); - } + } + + /** + * Test P_11 (l=1, m=1) + */ + @Test + @DisplayName("Test P11") + public void testP11() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(1, 1); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(-Math.sqrt(1 - Math.pow(x, 2)), alp.value(x), 1e-8); } - - /** - * Test P_21 (l=2, m=1) - */ - @Test - @DisplayName("Test P21") - public void testP21() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2,1); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(-3.0*x*Math.sqrt(1-Math.pow(x,2)), alp.value(x), 1e-8); - } + } + + /** + * Test P_20 (l=2, m=0) + */ + @Test + @DisplayName("Test P20") + public void testP20() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2, 0); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(0.5 * (3 * Math.pow(x, 2) - 1), alp.value(x), 1e-8); } - - /** - * Test P_21 (l=2, m=2) - */ - @Test - @DisplayName("Test P22") - public void testP22() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2,2); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(3.0*(1-Math.pow(x,2)), alp.value(x), 1e-8); - } + } + + /** + * Test P_21 (l=2, m=1) + */ + @Test + @DisplayName("Test P21") + public void testP21() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2, 1); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(-3.0 * x * Math.sqrt(1 - Math.pow(x, 2)), alp.value(x), 1e-8); } - - /** - * Test P_30 (l=3, m=0) - */ - @Test - @DisplayName("Test P30") - public void testP30() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3,0); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(0.5*(5.0*Math.pow(x,3) - 3.0*x), alp.value(x), 1e-8); - } + } + + /** + * Test P_21 (l=2, m=2) + */ + @Test + @DisplayName("Test P22") + public void testP22() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(2, 2); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(3.0 * (1 - Math.pow(x, 2)), alp.value(x), 1e-8); } - - /** - * Test P_30 (l=3, m=1) - */ - @Test - @DisplayName("Test P31") - public void testP31() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3,1); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(-(3.0/2.0)*(5.0*Math.pow(x,2) - 1.0)*Math.sqrt((1-Math.pow(x,2))), alp.value(x), 1e-8); - } + } + + /** + * Test P_30 (l=3, m=0) + */ + @Test + @DisplayName("Test P30") + public void testP30() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3, 0); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(0.5 * (5.0 * Math.pow(x, 3) - 3.0 * x), alp.value(x), 1e-8); } - - /** - * Test P_30 (l=3, m=2) - */ - @Test - @DisplayName("Test P32") - public void testP32() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3,2); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(15*x*(1-Math.pow(x,2)), alp.value(x), 1e-8); - } + } + + /** + * Test P_30 (l=3, m=1) + */ + @Test + @DisplayName("Test P31") + public void testP31() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3, 1); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(-(3.0 / 2.0) * (5.0 * Math.pow(x, 2) - 1.0) * Math.sqrt((1 - Math.pow(x, 2))), alp.value(x), 1e-8); } - - /** - * Test P_30 (l=3, m=3) - */ - @Test - @DisplayName("Test P33") - public void testP33() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3,3); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals(-15*Math.pow(1-Math.pow(x,2),3.0/2.0), alp.value(x), 1e-8); - } + } + + /** + * Test P_30 (l=3, m=2) + */ + @Test + @DisplayName("Test P32") + public void testP32() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3, 2); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(15 * x * (1 - Math.pow(x, 2)), alp.value(x), 1e-8); } - - /** - * Test P_40 (l=4, m=0) - */ - @Test - @DisplayName("Test P40") - public void testP40() { - final double increment = 1e-4; - final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(4,0); - for (double x = -1.0;x<=1.0;x+=increment) { - assertEquals((1.0/8.0)*(35*Math.pow(x,4) - 30*Math.pow(x,2) + 3), alp.value(x), 1e-8); - } + } + + /** + * Test P_30 (l=3, m=3) + */ + @Test + @DisplayName("Test P33") + public void testP33() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(3, 3); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals(-15 * Math.pow(1 - Math.pow(x, 2), 3.0 / 2.0), alp.value(x), 1e-8); + } + } + + /** + * Test P_40 (l=4, m=0) + */ + @Test + @DisplayName("Test P40") + public void testP40() { + final double increment = 1e-4; + final AssociatedLegendrePolynomial alp = new AssociatedLegendrePolynomial(4, 0); + for (double x = -1.0; x <= 1.0; x += increment) { + assertEquals((1.0 / 8.0) * (35 * Math.pow(x, 4) - 30 * Math.pow(x, 2) + 3), alp.value(x), 1e-8); } + } } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/RadialPolynomialTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/RadialPolynomialTest.java index 2e0dd39ec..5c588d30e 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/RadialPolynomialTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/RadialPolynomialTest.java @@ -1,117 +1,118 @@ package org.vitrivr.cineast.core.util.math.functions; +import static org.junit.jupiter.api.Assertions.assertEquals; + import org.apache.commons.math3.analysis.polynomials.PolynomialFunction; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.vitrivr.cineast.core.util.MathHelper; import org.vitrivr.cineast.core.util.math.functions.factories.PolynomialFunctionFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class RadialPolynomialTest { - /** - * Tests Orthogonality relation that exists between two radial polynomials. - */ - @Test - @DisplayName("Test Orthogonality") - void testOrthogonality() { - for (int n1=0;n1<10;n1++) { - for (int n2=0;n2<10;n2++) { - int max_m = Math.min(n1, n2); - for (int m=0;m<=max_m;m++) { - PolynomialFunction R1 = PolynomialFunctionFactory.createRadialPolynomial(n1, m); - PolynomialFunction R2 = PolynomialFunctionFactory.createRadialPolynomial(n2, m); - double result = 0.0; - double increment = 1e-6; - for (double d=0.0;d<=1.0f;d+=increment) { - result += R1.value(d) * R2.value(d) * d * increment; - } - assertEquals((double)MathHelper.kronecker(n1,n2)/(2*n1 + 2) * R1.value(1.0), result, 1e-2); - } - } + + /** + * Tests Orthogonality relation that exists between two radial polynomials. + */ + @Test + @DisplayName("Test Orthogonality") + void testOrthogonality() { + for (int n1 = 0; n1 < 10; n1++) { + for (int n2 = 0; n2 < 10; n2++) { + int max_m = Math.min(n1, n2); + for (int m = 0; m <= max_m; m++) { + PolynomialFunction R1 = PolynomialFunctionFactory.createRadialPolynomial(n1, m); + PolynomialFunction R2 = PolynomialFunctionFactory.createRadialPolynomial(n2, m); + double result = 0.0; + double increment = 1e-6; + for (double d = 0.0; d <= 1.0f; d += increment) { + result += R1.value(d) * R2.value(d) * d * increment; + } + assertEquals((double) MathHelper.kronecker(n1, n2) / (2 * n1 + 2) * R1.value(1.0), result, 1e-2); } + } } + } - /** - * Tests the R_nn (m=n) cases for the radial polynomials from n=0 to n=20. - */ - @Test - @DisplayName("Test R_nn") - void testRnn() { - for (int n=0;n<20;n++) { - PolynomialFunction Rnn = PolynomialFunctionFactory.createRadialPolynomial(n, n); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(Math.pow(i,n), Rnn.value(i), 1e-8); - } - } + /** + * Tests the R_nn (m=n) cases for the radial polynomials from n=0 to n=20. + */ + @Test + @DisplayName("Test R_nn") + void testRnn() { + for (int n = 0; n < 20; n++) { + PolynomialFunction Rnn = PolynomialFunctionFactory.createRadialPolynomial(n, n); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(Math.pow(i, n), Rnn.value(i), 1e-8); + } } + } - /** - * Tests the values of the R_10 (n=1, m=0) polynomial), which are supposed to be zero. - */ - @Test - @DisplayName("Test R_10") - void testR10() { - for (int n=0;n<20;n++) { - PolynomialFunction Rnn = PolynomialFunctionFactory.createRadialPolynomial(1, 0); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(0.0, Rnn.value(i), 1e-8); - } - } + /** + * Tests the values of the R_10 (n=1, m=0) polynomial), which are supposed to be zero. + */ + @Test + @DisplayName("Test R_10") + void testR10() { + for (int n = 0; n < 20; n++) { + PolynomialFunction Rnn = PolynomialFunctionFactory.createRadialPolynomial(1, 0); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(0.0, Rnn.value(i), 1e-8); + } } + } - /** - * Tests the values of the R_40 (n=4, m=0) polynomial). - */ - @Test - @DisplayName("Test R_40") - void testR04() { - PolynomialFunction R40 = PolynomialFunctionFactory.createRadialPolynomial(4, 0); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(6*Math.pow(i,4) - 6*Math.pow(i,2) + 1, R40.value(i), 1e-8); - } + /** + * Tests the values of the R_40 (n=4, m=0) polynomial). + */ + @Test + @DisplayName("Test R_40") + void testR04() { + PolynomialFunction R40 = PolynomialFunctionFactory.createRadialPolynomial(4, 0); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(6 * Math.pow(i, 4) - 6 * Math.pow(i, 2) + 1, R40.value(i), 1e-8); } + } - /** - * Tests the values of the R_51 (n=5, m=1) polynomial). - */ - @Test - @DisplayName("Test R_51") - void testR51() { - PolynomialFunction R52N = PolynomialFunctionFactory.createRadialPolynomial(5, -1); - PolynomialFunction R52P = PolynomialFunctionFactory.createRadialPolynomial(5, 1); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(10*Math.pow(i,5) - 12*Math.pow(i,3) + 3*i, R52N.value(i), 1e-8); - assertEquals(10*Math.pow(i,5) - 12*Math.pow(i,3) + 3*i, R52P.value(i), 1e-8); + /** + * Tests the values of the R_51 (n=5, m=1) polynomial). + */ + @Test + @DisplayName("Test R_51") + void testR51() { + PolynomialFunction R52N = PolynomialFunctionFactory.createRadialPolynomial(5, -1); + PolynomialFunction R52P = PolynomialFunctionFactory.createRadialPolynomial(5, 1); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(10 * Math.pow(i, 5) - 12 * Math.pow(i, 3) + 3 * i, R52N.value(i), 1e-8); + assertEquals(10 * Math.pow(i, 5) - 12 * Math.pow(i, 3) + 3 * i, R52P.value(i), 1e-8); - } } + } - /** - * Tests the values of the R_53 (n=5, m=3) polynomial). - */ - @Test - @DisplayName("Test R_53") - void testR53() { - PolynomialFunction R53P = PolynomialFunctionFactory.createRadialPolynomial(5, 3); - PolynomialFunction R53N = PolynomialFunctionFactory.createRadialPolynomial(5, -3); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(5*Math.pow(i,5) - 4*Math.pow(i,3), R53P.value(i), 1e-8); - assertEquals(5*Math.pow(i,5) - 4*Math.pow(i,3), R53N.value(i), 1e-8); + /** + * Tests the values of the R_53 (n=5, m=3) polynomial). + */ + @Test + @DisplayName("Test R_53") + void testR53() { + PolynomialFunction R53P = PolynomialFunctionFactory.createRadialPolynomial(5, 3); + PolynomialFunction R53N = PolynomialFunctionFactory.createRadialPolynomial(5, -3); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(5 * Math.pow(i, 5) - 4 * Math.pow(i, 3), R53P.value(i), 1e-8); + assertEquals(5 * Math.pow(i, 5) - 4 * Math.pow(i, 3), R53N.value(i), 1e-8); - } } + } - /** - * Tests the values of the R_60 (n=6, m=0) polynomial). - */ - @Test - @DisplayName("Test R_60") - void testR60() { - PolynomialFunction R40 = PolynomialFunctionFactory.createRadialPolynomial(6, 0); - for (double i = -2.0f; i < 2.0f; i += 0.01) { - assertEquals(20*Math.pow(i,6) - 30*Math.pow(i,4) + 12*Math.pow(i,2) - 1, R40.value(i), 1e-8); - } + /** + * Tests the values of the R_60 (n=6, m=0) polynomial). + */ + @Test + @DisplayName("Test R_60") + void testR60() { + PolynomialFunction R40 = PolynomialFunctionFactory.createRadialPolynomial(6, 0); + for (double i = -2.0f; i < 2.0f; i += 0.01) { + assertEquals(20 * Math.pow(i, 6) - 30 * Math.pow(i, 4) + 12 * Math.pow(i, 2) - 1, R40.value(i), 1e-8); } + } } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/ZernikePolynomialsTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/ZernikePolynomialsTest.java index 7f3dc8844..d5577c020 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/ZernikePolynomialsTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/util/math/functions/ZernikePolynomialsTest.java @@ -1,92 +1,89 @@ package org.vitrivr.cineast.core.util.math.functions; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.commons.math3.complex.Complex; import org.apache.commons.math3.util.FastMath; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.vitrivr.cineast.core.util.MathHelper; -import org.vitrivr.cineast.core.util.math.functions.ZernikeBasisFunction; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; public class ZernikePolynomialsTest { - /** - * Tests definition of Zernike Polynomials, which are only defined - * for n > 0, |m| <= n and n-|m| = even - */ - @Test - @DisplayName("Test Definition") - void testDefinition() { - for (int n=-20;n<=20;n++) { - for (int m=-2*n; m<=2*n;m++) { - if (((n-Math.abs(m)) % 2 == 1) || n < 0 || Math.abs(m) > n) { - final int n_u = n; - final int m_u = m; - assertThrows(IllegalArgumentException.class, () -> { - new ZernikeBasisFunction(n_u,m_u); - }); - } - } + + /** + * Tests definition of Zernike Polynomials, which are only defined for n > 0, |m| <= n and n-|m| = even + */ + @Test + @DisplayName("Test Definition") + void testDefinition() { + for (int n = -20; n <= 20; n++) { + for (int m = -2 * n; m <= 2 * n; m++) { + if (((n - Math.abs(m)) % 2 == 1) || n < 0 || Math.abs(m) > n) { + final int n_u = n; + final int m_u = m; + assertThrows(IllegalArgumentException.class, () -> { + new ZernikeBasisFunction(n_u, m_u); + }); } + } } + } - /** - * Tests if orthogonality relation that exists between two Zernike Polynoms holds true - * for all n between 1 and 5. - */ - @Test - @DisplayName("Test Orthogonality") - void testOrthogonality() { - final double increment = 0.25e-2; - final double n_max = 5; - for (int n1=1;n1<=n_max;n1++) { - for (int m1=0; m1<=n1;m1++) { - for (int n2=1;n2<=n_max;n2++) { - for (int m2=0;m2<=n2;m2++) { - if (((n1-Math.abs(m1)) % 2 == 0) && ((n2-Math.abs(m2)) % 2 == 0)) { - Complex result = new Complex(0, 0); - - /* Initialize ZernikeBasisFunctions for n1,m1 and n2,m2. */ - final ZernikeBasisFunction ZF1 = new ZernikeBasisFunction(n1, m1); - final ZernikeBasisFunction ZF2 = new ZernikeBasisFunction(n2, m2); - final double expected = ((Math.PI) / (n1 + 1)) * MathHelper.kronecker(n1,n2) * MathHelper.kronecker(m1,m2); + /** + * Tests if orthogonality relation that exists between two Zernike Polynoms holds true for all n between 1 and 5. + */ + @Test + @DisplayName("Test Orthogonality") + void testOrthogonality() { + final double increment = 0.25e-2; + final double n_max = 5; + for (int n1 = 1; n1 <= n_max; n1++) { + for (int m1 = 0; m1 <= n1; m1++) { + for (int n2 = 1; n2 <= n_max; n2++) { + for (int m2 = 0; m2 <= n2; m2++) { + if (((n1 - Math.abs(m1)) % 2 == 0) && ((n2 - Math.abs(m2)) % 2 == 0)) { + Complex result = new Complex(0, 0); - /* Calculate integral (approximation). */ - for (double theta = 0.0; theta <= 2 * Math.PI; theta += increment) { - for (double r = 0.0; r <= 1.0f; r += increment) { - Complex v = new Complex(r * FastMath.cos(theta), r * FastMath.sin(theta)); - Complex res1 = ZF1.value(v); - Complex res2 = ZF2.value(v); - result = result.add(res1.conjugate().multiply(res2).multiply(r * increment * increment)); - } - } + /* Initialize ZernikeBasisFunctions for n1,m1 and n2,m2. */ + final ZernikeBasisFunction ZF1 = new ZernikeBasisFunction(n1, m1); + final ZernikeBasisFunction ZF2 = new ZernikeBasisFunction(n2, m2); + final double expected = ((Math.PI) / (n1 + 1)) * MathHelper.kronecker(n1, n2) * MathHelper.kronecker(m1, m2); - /* Result of integral must be equal to expected value. */ - assertEquals(expected, result.abs(), 1e-2); - } - } + /* Calculate integral (approximation). */ + for (double theta = 0.0; theta <= 2 * Math.PI; theta += increment) { + for (double r = 0.0; r <= 1.0f; r += increment) { + Complex v = new Complex(r * FastMath.cos(theta), r * FastMath.sin(theta)); + Complex res1 = ZF1.value(v); + Complex res2 = ZF2.value(v); + result = result.add(res1.conjugate().multiply(res2).multiply(r * increment * increment)); } + } + + /* Result of integral must be equal to expected value. */ + assertEquals(expected, result.abs(), 1e-2); } + } } + } } + } - /** - * Test Z_00, which must be equal to 1.0 + 0.0i for all values ofs r - * and theta. - */ - @Test - @DisplayName("Test Z_00") - void testZ00() { - final ZernikeBasisFunction ZF1 = new ZernikeBasisFunction(0, 0); - final double increment = 1e-3; - for (double r = 0.0; r <= 1.0f; r += increment) { - for (double theta = 0; theta <= 2*Math.PI; theta += increment) { - Complex v = new Complex(r * FastMath.cos(theta), r * FastMath.sin(theta)); - assertEquals(1.0, ZF1.value(v).abs(), 1e-8); - assertEquals(0.0, ZF1.value(v).getArgument(), 1e-8); - } - } + /** + * Test Z_00, which must be equal to 1.0 + 0.0i for all values ofs r and theta. + */ + @Test + @DisplayName("Test Z_00") + void testZ00() { + final ZernikeBasisFunction ZF1 = new ZernikeBasisFunction(0, 0); + final double increment = 1e-3; + for (double r = 0.0; r <= 1.0f; r += increment) { + for (double theta = 0; theta <= 2 * Math.PI; theta += increment) { + Complex v = new Complex(r * FastMath.cos(theta), r * FastMath.sin(theta)); + assertEquals(1.0, ZF1.value(v).abs(), 1e-8); + assertEquals(0.0, ZF1.value(v).getArgument(), 1e-8); + } } + } } diff --git a/cineast-runtime/build.gradle b/cineast-runtime/build.gradle index 4bf70228d..306dc3535 100644 --- a/cineast-runtime/build.gradle +++ b/cineast-runtime/build.gradle @@ -17,7 +17,7 @@ distributions { } } -jar{ +jar { manifest { attributes 'Main-Class': 'org.vitrivr.cineast.standalone.Main' attributes 'Multi-Release': 'true' @@ -41,7 +41,7 @@ dependencies { /** Airline (CLI parsing) */ api group: 'com.github.rvesse', name: 'airline', version: version_airline - + /** JLine 3 */ api group: 'org.jline', name: 'jline', version: version_jline diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java index 26a1ecc0d..3a617977b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java @@ -23,11 +23,11 @@ public static void main(String[] args) { /* Initalize Monitoring */ PrometheusServer.initialize(); - if(args.length==1){ + if (args.length == 1) { CLI.start(CineastCli.class); } /* Either start Cineast in interactive mode OR execute command directly. */ - if(args.length==2 && args[1].equals("interactive")){ + if (args.length == 2 && args[1].equals("interactive")) { CLI.start(CineastCli.class); } else { com.github.rvesse.airline.Cli cli = new com.github.rvesse.airline.Cli<>(CineastCli.class); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java index 9e5997ae0..afa020d11 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java @@ -69,7 +69,7 @@ public static void printInfoForSegment(String segmentId, DBSelector selector, St retriever.getTableNames().forEach(tableName -> { selector.open(tableName); List> rows = selector.getRows("id", new StringTypeProvider(segmentId)); - if( retriever.getClass() == RangeBooleanRetriever.class){ + if (retriever.getClass() == RangeBooleanRetriever.class) { rows = selector.getRows("segmentid", new StringTypeProvider(segmentId)); } rows.forEach(row -> { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CodebookCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CodebookCommand.java index 0b237db10..92b24bbbf 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CodebookCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CodebookCommand.java @@ -10,7 +10,6 @@ /** * A CLI command that can be used to generate SURF or HOG codebooks based on a set of images. - * */ @Command(name = "codebook", description = "Generates a codebook of defined size based on a set of images using some specified features.") public class CodebookCommand implements Runnable { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DatabaseSetupCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DatabaseSetupCommand.java index 757fa0f71..3a8714193 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DatabaseSetupCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DatabaseSetupCommand.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.PersistentOperator; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.config.ExtractorConfig; @@ -18,7 +17,6 @@ /** * A CLI command that can be used to setup all the database entities required by Cineast. - * */ @Command(name = "setup", description = "Makes the necessary database setup for Cineast and creates all the required entities and inidices.") public class DatabaseSetupCommand implements Runnable { @@ -57,7 +55,7 @@ private HashSet extractionConfigPersistentOperators(String e /** * This is a hacky way to support CLI and non CLI usage. - * + *

    * TL;DR The functionality of this class is used in a non-cottontail configuration when clean-before-import is used. See {@link org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler} for further explanation. */ private final boolean isNotCommand; @@ -127,7 +125,7 @@ public void doSetup() { /** * Drops all entities currently required by Cineast. * - * @param ec The {@link EntityCreator} used to drop the entities. + * @param ec The {@link EntityCreator} used to drop the entities. * @param persistentOperators The list of {@link PersistentOperator} classes to drop the entities for. */ private void dropAllEntities(EntityCreator ec, Collection persistentOperators) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java index 926fbedfa..5266b14fc 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java @@ -108,7 +108,7 @@ public void extractionComplete() { /** * Configures an IIIF extraction job by downloading all specified images from the server onto to the filesystem and pointing the {@link ExtractionContainerProvider} to that directory. * - * @param iiifConfig The IIIF config parsed as an {@link IIIFConfig} + * @param iiifConfig The IIIF config parsed as an {@link IIIFConfig} * @param directoryPath The path where the downloaded IIIF content should be stored * @throws IOException Thrown if downloading or writing an image or it's associated information encounters an IOException */ diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java index 143311862..81f485aec 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java @@ -2,15 +2,13 @@ import com.github.rvesse.airline.annotations.Command; import com.github.rvesse.airline.annotations.Option; +import com.github.rvesse.airline.annotations.restrictions.Required; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; - -import com.github.rvesse.airline.annotations.restrictions.Required; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cineast.standalone.importer.lsc2020.MyscealTagImportHandler; import org.vitrivr.cineast.standalone.importer.handlers.AsrDataImportHandler; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; import org.vitrivr.cineast.standalone.importer.handlers.JsonDataImportHandler; @@ -20,6 +18,7 @@ import org.vitrivr.cineast.standalone.importer.lsc2020.CaptionImportHandler; import org.vitrivr.cineast.standalone.importer.lsc2020.LSCAllTagsImportHandler; import org.vitrivr.cineast.standalone.importer.lsc2020.MetaImportHandler; +import org.vitrivr.cineast.standalone.importer.lsc2020.MyscealTagImportHandler; import org.vitrivr.cineast.standalone.importer.lsc2020.OCRImportHandler; import org.vitrivr.cineast.standalone.importer.lsc2020.ProcessingMetaImportHandler; import org.vitrivr.cineast.standalone.importer.lsc2020.ProcessingMetaImportHandler.Mode; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java index d2753f237..975f5324b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.standalone.cli; import com.github.rvesse.airline.annotations.Command; - import org.vitrivr.cineast.core.config.DatabaseConfig.Selector; import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/RetrieveCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/RetrieveCommand.java index 240d5fb5c..8ba89451e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/RetrieveCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/RetrieveCommand.java @@ -33,7 +33,6 @@ public void run() { retrieval.addRetrievalResultListener(new RetrievalResultCSVExporter(Config.sharedConfig().getDatabase())); } - QueryConfig qc = QueryConfig.newQueryConfigFromOther(new ConstrainedQueryConfig("cli-query", new ArrayList<>())); if (relevantSegments != null && !relevantSegments.isEmpty()) { @@ -46,7 +45,6 @@ public void run() { qc.setMaxResults(Config.sharedConfig().getRetriever().getMaxResults()); qc.setResultsPerModule(Config.sharedConfig().getRetriever().getMaxResultsPerModule()); - final List results = retrieval.retrieve(this.segmentId, this.category, qc); System.out.println("results:"); results.forEach(System.out::println); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java index 0d6279be3..6d06909f8 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java @@ -6,7 +6,6 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -25,7 +24,6 @@ import org.vitrivr.cottontail.client.language.dml.Update; import org.vitrivr.cottontail.client.language.dql.Query; import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; -import org.vitrivr.cottontail.grpc.CottontailGrpc.ComparisonOperator; import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal.Builder; import org.vitrivr.cottontail.grpc.CottontailGrpc.UpdateMessage.UpdateElement; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java index 569cd2cad..daaf12737 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.standalone.config; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java index 34c19104c..485801759 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java @@ -2,48 +2,52 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.File; +import java.util.HashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.*; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.data.raw.CachedDataFactory; import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; -import java.io.File; -import java.util.HashMap; - @JsonIgnoreProperties(ignoreUnknown = true) public class Config { - private static final Logger LOGGER = LogManager.getLogger(); - - /** Global, shared instance of the Config object. Gets loading during application startup. */ - private volatile static Config sharedConfig; - - private APIConfig api; - private DatabaseConfig database; - private RetrievalRuntimeConfig retriever; - private ExtractionPipelineConfig extractor; - private CacheConfig cache; - private HashMap decoders; - private MonitoringConfig monitoring = new MonitoringConfig(); - - /** - * Accessor for shared (i.e. application wide) configuration. - * - * @return Currently shared instance of Config. - */ - public synchronized static Config sharedConfig() { - if (sharedConfig == null) { - loadConfig("cineast.json"); - } - return sharedConfig; - } - /** - * Loads a config file and thereby replaces the shared instance of the Config. - * - * @param name Name of the config file. - */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Global, shared instance of the Config object. Gets loading during application startup. + */ + private volatile static Config sharedConfig; + + private APIConfig api; + private DatabaseConfig database; + private RetrievalRuntimeConfig retriever; + private ExtractionPipelineConfig extractor; + private CacheConfig cache; + private HashMap decoders; + private MonitoringConfig monitoring = new MonitoringConfig(); + + /** + * Accessor for shared (i.e. application wide) configuration. + * + * @return Currently shared instance of Config. + */ + public synchronized static Config sharedConfig() { + if (sharedConfig == null) { + loadConfig("cineast.json"); + } + return sharedConfig; + } + + /** + * Loads a config file and thereby replaces the shared instance of the Config. + * + * @param name Name of the config file. + */ public static Config loadConfig(String name) { final Config config = (new JacksonJsonProvider()).toObject(new File(name), Config.class); if (config == null) { @@ -64,63 +68,70 @@ public static void initSharedConfig(Config config) { } - @JsonProperty - public APIConfig getApi() { - return api; - } - public void setApi(APIConfig api) { - this.api = api; - } + @JsonProperty + public APIConfig getApi() { + return api; + } - @JsonProperty - public DatabaseConfig getDatabase() { - return database; - } - public void setDatabase(DatabaseConfig database) { - this.database = database; - } + public void setApi(APIConfig api) { + this.api = api; + } - @JsonProperty - public RetrievalRuntimeConfig getRetriever() { - return retriever; - } - public void setRetriever(RetrievalRuntimeConfig retriever) { - this.retriever = retriever; - } + @JsonProperty + public DatabaseConfig getDatabase() { + return database; + } - @JsonProperty - public ExtractionPipelineConfig getExtractor() { - return extractor; - } - public void setExtractor(ExtractionPipelineConfig extractor) { - this.extractor = extractor; - } + public void setDatabase(DatabaseConfig database) { + this.database = database; + } - @JsonProperty - public CacheConfig getCache() { - if(cache == null){ - cache = new CacheConfig(); - } - return cache; - } - public void setCache(CacheConfig cache) { - this.cache = cache; - } + @JsonProperty + public RetrievalRuntimeConfig getRetriever() { + return retriever; + } + public void setRetriever(RetrievalRuntimeConfig retriever) { + this.retriever = retriever; + } - @JsonProperty - public HashMap getDecoders() { - return decoders; - } - public void setDecoders(HashMap decoders) { - this.decoders = decoders; - } + @JsonProperty + public ExtractionPipelineConfig getExtractor() { + return extractor; + } - @JsonProperty - public MonitoringConfig getMonitoring() { - return monitoring; - } - public void setMonitoring(MonitoringConfig monitoring) { - this.monitoring = monitoring; + public void setExtractor(ExtractionPipelineConfig extractor) { + this.extractor = extractor; + } + + @JsonProperty + public CacheConfig getCache() { + if (cache == null) { + cache = new CacheConfig(); } + return cache; + } + + public void setCache(CacheConfig cache) { + this.cache = cache; + } + + + @JsonProperty + public HashMap getDecoders() { + return decoders; + } + + public void setDecoders(HashMap decoders) { + this.decoders = decoders; + } + + @JsonProperty + public MonitoringConfig getMonitoring() { + return monitoring; + } + + public void setMonitoring(MonitoringConfig monitoring) { + this.monitoring = monitoring; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ConstrainedQueryConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ConstrainedQueryConfig.java index 2c088969b..dca8c2f2f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ConstrainedQueryConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ConstrainedQueryConfig.java @@ -1,36 +1,36 @@ package org.vitrivr.cineast.standalone.config; -import org.vitrivr.cineast.core.config.QueryConfig; -import org.vitrivr.cineast.core.config.ReadableQueryConfig; - import java.util.List; import java.util.Optional; +import org.vitrivr.cineast.core.config.QueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; public class ConstrainedQueryConfig extends QueryConfig { - public ConstrainedQueryConfig(String queryId, List hints) { - super(queryId, hints); - } + public ConstrainedQueryConfig(String queryId, List hints) { + super(queryId, hints); + } - public ConstrainedQueryConfig(ReadableQueryConfig qc) { - super(qc); - } + public ConstrainedQueryConfig(ReadableQueryConfig qc) { + super(qc); + } - public ConstrainedQueryConfig() { - super(null); - } + public ConstrainedQueryConfig() { + super(null); + } - @Override - public int getResultsPerModule() { return Math.min(Config.sharedConfig().getRetriever().getMaxResultsPerModule(), super.getResultsPerModule()); } + @Override + public int getResultsPerModule() { + return Math.min(Config.sharedConfig().getRetriever().getMaxResultsPerModule(), super.getResultsPerModule()); + } - @Override - public Optional getMaxResults() { - if (super.getMaxResults().isPresent()) { - return Optional.of(Math.min(Config.sharedConfig().getRetriever().getMaxResults(), super.getMaxResults().get())); - } - return Optional.of(Config.sharedConfig().getRetriever().getMaxResults()); + @Override + public Optional getMaxResults() { + if (super.getMaxResults().isPresent()) { + return Optional.of(Math.min(Config.sharedConfig().getRetriever().getMaxResults(), super.getMaxResults().get())); } - + return Optional.of(Config.sharedConfig().getRetriever().getMaxResults()); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExplorativeConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExplorativeConfig.java index 8d21df65e..6a38662c5 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExplorativeConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExplorativeConfig.java @@ -1,7 +1,6 @@ package org.vitrivr.cineast.standalone.config; import com.fasterxml.jackson.annotation.JsonCreator; - import java.io.File; import java.io.FileInputStream; import java.util.Properties; @@ -9,63 +8,63 @@ public class ExplorativeConfig { - private static String csvPath = "/Users/silvanstich/Library/Containers/com.apple.mail/Data/Library/Mail Downloads/CA01A1C7-162D-454F-9645-8F9250035697/neuralnet_vgg16_fullvector"; - private static String featureName = "nn_vecotrs"; - private static String elementLimit = "1000"; - private static String dataFolder = "data/"; - private static String resultFolder = "results/html/experimental/"; - private static String treeSerializationFileName = "nn_serialized_tree.ser"; - private static String mode = "csv"; + private static String csvPath = "/Users/silvanstich/Library/Containers/com.apple.mail/Data/Library/Mail Downloads/CA01A1C7-162D-454F-9645-8F9250035697/neuralnet_vgg16_fullvector"; + private static String featureName = "nn_vecotrs"; + private static String elementLimit = "1000"; + private static String dataFolder = "data/"; + private static String resultFolder = "results/html/experimental/"; + private static String treeSerializationFileName = "nn_serialized_tree.ser"; + private static String mode = "csv"; - @JsonCreator - public ExplorativeConfig() { + @JsonCreator + public ExplorativeConfig() { - } + } - public static void readConfig(String file){ - Properties properties = new Properties(); - File propertiesFile = new File(file); - try { - FileInputStream fileInputStream = new FileInputStream(propertiesFile); - properties.load(fileInputStream); - fileInputStream.close(); - } catch (java.io.IOException e) { - return; // use hard coded defaults defaults - } - csvPath = properties.getProperty("csvPath", csvPath); - featureName = properties.getProperty("featureName", featureName); - elementLimit = properties.getProperty("elementLimit", elementLimit); - dataFolder = properties.getProperty("dataFolder", dataFolder); - resultFolder = properties.getProperty("resultFolder", resultFolder); - treeSerializationFileName = properties.getProperty("treeSerializationFileName", treeSerializationFileName); - mode = properties.getProperty("mode", mode); + public static void readConfig(String file) { + Properties properties = new Properties(); + File propertiesFile = new File(file); + try { + FileInputStream fileInputStream = new FileInputStream(propertiesFile); + properties.load(fileInputStream); + fileInputStream.close(); + } catch (java.io.IOException e) { + return; // use hard coded defaults defaults } + csvPath = properties.getProperty("csvPath", csvPath); + featureName = properties.getProperty("featureName", featureName); + elementLimit = properties.getProperty("elementLimit", elementLimit); + dataFolder = properties.getProperty("dataFolder", dataFolder); + resultFolder = properties.getProperty("resultFolder", resultFolder); + treeSerializationFileName = properties.getProperty("treeSerializationFileName", treeSerializationFileName); + mode = properties.getProperty("mode", mode); + } - public static String getCsvPath() { - return csvPath; - } + public static String getCsvPath() { + return csvPath; + } - public static String getFeatureName() { - return featureName; - } + public static String getFeatureName() { + return featureName; + } - public static String getElementLimit() { - return elementLimit; - } + public static String getElementLimit() { + return elementLimit; + } - public static String getDataFolder() { - return dataFolder; - } + public static String getDataFolder() { + return dataFolder; + } - public static String getResultFolder() { - return resultFolder; - } + public static String getResultFolder() { + return resultFolder; + } - public static String getTreeSerializationFileName() { - return treeSerializationFileName; - } + public static String getTreeSerializationFileName() { + return treeSerializationFileName; + } - public static String getMode() { - return mode; - } + public static String getMode() { + return mode; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java index d6afe71e5..3ec402560 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java @@ -2,64 +2,79 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.io.File; public final class ExtractionPipelineConfig { - /** Default value for size of thread-pool. */ - public static final int DEFAULT_THREADPOOL_SIZE = 4; - - /** Default value for size of task-queue. */ - public static final int DEFAULT_TASKQUEUE_SIZE = 10; - - /** Default value for size of segment-queue. */ - public static final int DEFAULT_SEGMENTQUEUE_SIZE = 10; - - /** */ - private Integer shotQueueSize = DEFAULT_SEGMENTQUEUE_SIZE; - - /** */ - private Integer threadPoolSize = DEFAULT_THREADPOOL_SIZE; - - /** */ - private Integer taskQueueSize = DEFAULT_TASKQUEUE_SIZE; - - private File outputLocation = new File("."); - - @JsonCreator - public ExtractionPipelineConfig() { - } - - @JsonProperty - public Integer getShotQueueSize(){ - return this.shotQueueSize; - } - public void setShotQueueSize(Integer shotQueueSize) { - this.shotQueueSize = shotQueueSize; - } - - @JsonProperty - public Integer getThreadPoolSize(){ - return this.threadPoolSize; - } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } - - @JsonProperty - public Integer getTaskQueueSize() { - return this.taskQueueSize; - } - public void setTaskQueueSize(int taskQueueSize) { - this.taskQueueSize = taskQueueSize; - } - - @JsonProperty - public File getOutputLocation(){ - return this.outputLocation; - } - public void setOutputLocation(String outputLocation) { - this.outputLocation = new File(outputLocation); - } + /** + * Default value for size of thread-pool. + */ + public static final int DEFAULT_THREADPOOL_SIZE = 4; + + /** + * Default value for size of task-queue. + */ + public static final int DEFAULT_TASKQUEUE_SIZE = 10; + + /** + * Default value for size of segment-queue. + */ + public static final int DEFAULT_SEGMENTQUEUE_SIZE = 10; + + /** + * + */ + private Integer shotQueueSize = DEFAULT_SEGMENTQUEUE_SIZE; + + /** + * + */ + private Integer threadPoolSize = DEFAULT_THREADPOOL_SIZE; + + /** + * + */ + private Integer taskQueueSize = DEFAULT_TASKQUEUE_SIZE; + + private File outputLocation = new File("."); + + @JsonCreator + public ExtractionPipelineConfig() { + } + + @JsonProperty + public Integer getShotQueueSize() { + return this.shotQueueSize; + } + + public void setShotQueueSize(Integer shotQueueSize) { + this.shotQueueSize = shotQueueSize; + } + + @JsonProperty + public Integer getThreadPoolSize() { + return this.threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + @JsonProperty + public Integer getTaskQueueSize() { + return this.taskQueueSize; + } + + public void setTaskQueueSize(int taskQueueSize) { + this.taskQueueSize = taskQueueSize; + } + + @JsonProperty + public File getOutputLocation() { + return this.outputLocation; + } + + public void setOutputLocation(String outputLocation) { + this.outputLocation = new File(outputLocation); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java index e9d34e20f..8db8d9fff 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java @@ -2,48 +2,52 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.HashMap; - public class ExtractorConfig { - /** Name of the Extractor. Must correspond to the simple-name or the FQN of the respective class. - * - * @see org.vitrivr.cineast.core.features.extractor.Extractor - */ - private String name; - - /** Properties that are being used to initialize the Extractor. - * - * @see org.vitrivr.cineast.core.features.extractor.Extractor - */ - private HashMap properties = new HashMap<>(); - - @JsonProperty(required = true) - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } - - @JsonProperty - public HashMap getProperties() { - return properties; - } - public void setProperties(HashMap properties) { - this.properties = properties; - } - - @JsonIgnore - public Extractor getExtractor() { - return ReflectionHelper.newExtractor(this.name); - } - - @JsonIgnore - public Extractor getExporter() { - return ReflectionHelper.newExporter(this.name, this.properties); - } + + /** + * Name of the Extractor. Must correspond to the simple-name or the FQN of the respective class. + * + * @see org.vitrivr.cineast.core.features.extractor.Extractor + */ + private String name; + + /** + * Properties that are being used to initialize the Extractor. + * + * @see org.vitrivr.cineast.core.features.extractor.Extractor + */ + private HashMap properties = new HashMap<>(); + + @JsonProperty(required = true) + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @JsonProperty + public HashMap getProperties() { + return properties; + } + + public void setProperties(HashMap properties) { + this.properties = properties; + } + + @JsonIgnore + public Extractor getExtractor() { + return ReflectionHelper.newExtractor(this.name); + } + + @JsonIgnore + public Extractor getExporter() { + return ReflectionHelper.newExporter(this.name, this.properties); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java index 45a0ea444..0b267e364 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java @@ -2,366 +2,387 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.config.IdConfig; -import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.config.SegmenterConfig; import org.vitrivr.cineast.core.data.MediaType; import org.vitrivr.cineast.core.db.DBSelectorSupplier; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.extraction.ExtractionContextProvider; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; +import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; import org.vitrivr.cineast.core.extraction.segmenter.general.Segmenter; import org.vitrivr.cineast.core.features.extractor.Extractor; -import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; import org.vitrivr.cineast.standalone.run.path.SessionContainerProvider; import org.vitrivr.cineast.standalone.run.path.SingletonContainerProvider; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - /** * Configures a data-ingest or extraction run, acts as an ExtractionContextProvider. - * - * A concrete instance can be obtained by deserializing a JSON file that is compatible with the - * structure defined by this classes and its fields. - * + *

    + * A concrete instance can be obtained by deserializing a JSON file that is compatible with the structure defined by this classes and its fields. */ public class IngestConfig implements ExtractionContextProvider { - /** MediaType for the Extraction run. */ - private final MediaType type; - - /** Input configuration for the Extraction run*/ - private final InputConfig input; - - /** List of {@link ExtractorConfig}s, one for every {@link Extractor} that should be used during the extraction run. */ - private final List extractors; - - /** List of {@link ExtractorConfig}s, one for every exporter that should be used during the extraction run. */ - private final List exporters; - - /** List of {@link MetadataConfig} entries, one for each {@link MetadataExtractor} that should be used during an extraction run. */ - private final List metadata; - - /** Database-setting to use for import. Defaults to application settings. */ - private final DatabaseConfig database; - - /** Configuration for extraction-pipeline. Defaults to global configuration. */ - private final ExtractionPipelineConfig pipeline; - - /** Configuration for extraction-pipeline. Defaults to global configuration. */ - private final SegmenterConfig segmenter; - - /** Configuration for extraction-pipeline. Defaults to global configuration. */ - private final CacheConfig cacheConfig; - - /** - * Constructor for {@link IngestConfig}. Used by Jackson for JSON deserialization. - * - */ - @JsonCreator - public IngestConfig(@JsonProperty(value = "type") MediaType type, - @JsonProperty(value = "input", required = true) InputConfig input, - @JsonProperty(value = "extractors") List extractors, - @JsonProperty(value = "exporters") List exporters, - @JsonProperty(value = "metadata") List metadata, - @JsonProperty(value = "database") DatabaseConfig database, - @JsonProperty(value = "pipeline") ExtractionPipelineConfig pipeline, - @JsonProperty(value = "segmenter") SegmenterConfig segmenter, - @JsonProperty(value = "imagecache") CacheConfig cacheConfig) { - - if (input == null) throw new IllegalArgumentException("You have not defined an 'type' or 'input' object in your ingest configuration file."); - this.type = type; - this.input = input; - - /* Initialize list of ExtractorConfig. */ - if (extractors == null) extractors = new ArrayList<>(0); - this.extractors = extractors; - - /* Initialize list of ExtractorConfigs. */ - if (exporters == null) exporters = new ArrayList<>(0); - this.exporters = exporters; - - /* Initialize list of MetadataExtractors. */ - if (metadata == null) metadata = new ArrayList<>(0); - this.metadata = metadata; - - /* Initialize DatabaseConfig. */ - final DatabaseConfig globalDb = Config.sharedConfig().getDatabase(); - if (database == null) database = globalDb; - this.database = database; - if (this.database.getSelector() == null) { - this.database.setSelector(globalDb.getSelector()); - } - if (this.database.getWriter() == null) { - this.database.setWriter(globalDb.getWriter()); - } - if (this.database.getBatchsize() == DatabaseConfig.DEFAULT_BATCH_SIZE) { - this.database.setBatchsize(globalDb.getBatchsize()); - } - if (this.database.getHost().equals(DatabaseConfig.DEFAULT_HOST)) { - this.database.setHost(globalDb.getHost()); - } - if (this.database.getPort() == DatabaseConfig.DEFAULT_PORT) { - this.database.setPort(globalDb.getPort()); - } - if (this.database.getPlaintext() == DatabaseConfig.DEFAULT_PLAINTEXT) { - this.database.setPlaintext(globalDb.getPlaintext()); - } - - /* Merge with global settings if not set. */ - final ExtractionPipelineConfig globalExt = Config.sharedConfig().getExtractor(); - if (pipeline == null) pipeline = globalExt; - this.pipeline = pipeline; - if (this.pipeline.getTaskQueueSize() == ExtractionPipelineConfig.DEFAULT_TASKQUEUE_SIZE) { - this.pipeline.setTaskQueueSize(globalExt.getTaskQueueSize()); - } - if (this.pipeline.getThreadPoolSize() == ExtractionPipelineConfig.DEFAULT_THREADPOOL_SIZE) { - this.pipeline.setThreadPoolSize(globalExt.getThreadPoolSize()); - } - if (this.pipeline.getShotQueueSize() == ExtractionPipelineConfig.DEFAULT_SEGMENTQUEUE_SIZE) { - this.pipeline.setShotQueueSize(globalExt.getShotQueueSize()); - } - - /* Set SegmenterConfig. */ - if (segmenter == null) segmenter = new SegmenterConfig(this.type); - this.segmenter = segmenter; - - /* Set ImageCacheConfig. */ - if (cacheConfig == null) cacheConfig = Config.sharedConfig().getCache(); - this.cacheConfig = cacheConfig; - } - @JsonProperty(required = true) - public InputConfig getInput() { - return input; + /** + * MediaType for the Extraction run. + */ + private final MediaType type; + + /** + * Input configuration for the Extraction run + */ + private final InputConfig input; + + /** + * List of {@link ExtractorConfig}s, one for every {@link Extractor} that should be used during the extraction run. + */ + private final List extractors; + + /** + * List of {@link ExtractorConfig}s, one for every exporter that should be used during the extraction run. + */ + private final List exporters; + + /** + * List of {@link MetadataConfig} entries, one for each {@link MetadataExtractor} that should be used during an extraction run. + */ + private final List metadata; + + /** + * Database-setting to use for import. Defaults to application settings. + */ + private final DatabaseConfig database; + + /** + * Configuration for extraction-pipeline. Defaults to global configuration. + */ + private final ExtractionPipelineConfig pipeline; + + /** + * Configuration for extraction-pipeline. Defaults to global configuration. + */ + private final SegmenterConfig segmenter; + + /** + * Configuration for extraction-pipeline. Defaults to global configuration. + */ + private final CacheConfig cacheConfig; + + /** + * Constructor for {@link IngestConfig}. Used by Jackson for JSON deserialization. + */ + @JsonCreator + public IngestConfig(@JsonProperty(value = "type") MediaType type, + @JsonProperty(value = "input", required = true) InputConfig input, + @JsonProperty(value = "extractors") List extractors, + @JsonProperty(value = "exporters") List exporters, + @JsonProperty(value = "metadata") List metadata, + @JsonProperty(value = "database") DatabaseConfig database, + @JsonProperty(value = "pipeline") ExtractionPipelineConfig pipeline, + @JsonProperty(value = "segmenter") SegmenterConfig segmenter, + @JsonProperty(value = "imagecache") CacheConfig cacheConfig) { + + if (input == null) { + throw new IllegalArgumentException("You have not defined an 'type' or 'input' object in your ingest configuration file."); } + this.type = type; + this.input = input; - @JsonProperty - public List getExtractors() { - return extractors; + /* Initialize list of ExtractorConfig. */ + if (extractors == null) { + extractors = new ArrayList<>(0); } + this.extractors = extractors; - @JsonProperty - public List getExporters() { - return exporters; + /* Initialize list of ExtractorConfigs. */ + if (exporters == null) { + exporters = new ArrayList<>(0); } + this.exporters = exporters; - @JsonProperty - public List getMetadata() { - return metadata; + /* Initialize list of MetadataExtractors. */ + if (metadata == null) { + metadata = new ArrayList<>(0); } + this.metadata = metadata; - @JsonProperty - public DatabaseConfig getDatabase() { - return database; + /* Initialize DatabaseConfig. */ + final DatabaseConfig globalDb = Config.sharedConfig().getDatabase(); + if (database == null) { + database = globalDb; } - - @JsonProperty - public ExtractionPipelineConfig getPipeline() { - return pipeline; + this.database = database; + if (this.database.getSelector() == null) { + this.database.setSelector(globalDb.getSelector()); } - - @JsonProperty - public SegmenterConfig getSegmenter() { - return this.segmenter; - } - - @Override - public Optional inputPath() { - if (this.input != null) { - return Optional.of(Paths.get(this.input.getPath())); - } else { - return Optional.empty(); - } + if (this.database.getWriter() == null) { + this.database.setWriter(globalDb.getWriter()); } - - @Override - public Optional relPath() { - if (this.input == null || this.input.getRelTo() == null) { - return Optional.empty(); - } - return Optional.of(Paths.get(this.input.getRelTo())); + if (this.database.getBatchsize() == DatabaseConfig.DEFAULT_BATCH_SIZE) { + this.database.setBatchsize(globalDb.getBatchsize()); } - - public ExtractionContainerProvider pathProvider() { - if (this.input != null) { - return new SingletonContainerProvider(Paths.get(this.input.getPath())); - } else { - return new SessionContainerProvider(); - } + if (this.database.getHost().equals(DatabaseConfig.DEFAULT_HOST)) { + this.database.setHost(globalDb.getHost()); } - - /** - * Determines the MediaType of the source material. Only one media-type - * can be specified per ExtractionContextProvider. - * - * @return Media-type of the source material. - */ - @Override - @JsonProperty - public MediaType getType() { - return this.type; + if (this.database.getPort() == DatabaseConfig.DEFAULT_PORT) { + this.database.setPort(globalDb.getPort()); } - - /** - * Returns a list of extractor classes that should be used for - * the extraction run! - * - * @return List of named extractors. - */ - @Override - public List extractors() { - return this.extractors.stream() - .map(ExtractorConfig::getExtractor) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + if (this.database.getPlaintext() == DatabaseConfig.DEFAULT_PLAINTEXT) { + this.database.setPlaintext(globalDb.getPlaintext()); } - /** - * Returns a list of exporter classes that should be invoked during extraction. Exporters - * usually generate some representation and persistently store that information somewhere. - * - * @return List of named exporters. - */ - @Override - public List exporters() { - return this.exporters.stream() - .map(ExtractorConfig::getExporter) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + /* Merge with global settings if not set. */ + final ExtractionPipelineConfig globalExt = Config.sharedConfig().getExtractor(); + if (pipeline == null) { + pipeline = globalExt; } - - /** - * Returns a list of metadata extractor classes that should be invoked during extraction. MetadataExtractor's - * usually read some metadata from a file. - * - * @return List of named exporters. - */ - @Override - public List metadataExtractors() { - return this.metadata.stream() - .map(MetadataConfig::getMetadataExtractor) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + this.pipeline = pipeline; + if (this.pipeline.getTaskQueueSize() == ExtractionPipelineConfig.DEFAULT_TASKQUEUE_SIZE) { + this.pipeline.setTaskQueueSize(globalExt.getTaskQueueSize()); } - - /** - * Selects, configures and returns a new instance of the {@link Segmenter} that was configured in the - * current instance of {@link ExtractionContextProvider}. - * - * @return {@link Segmenter} that was configured in the current instance of {@link ExtractionContextProvider} - */ - @Override - public Segmenter newSegmenter() { - return this.segmenter.newSegmenter(this); + if (this.pipeline.getThreadPoolSize() == ExtractionPipelineConfig.DEFAULT_THREADPOOL_SIZE) { + this.pipeline.setThreadPoolSize(globalExt.getThreadPoolSize()); } - - /** - * Limits the depth of recursion when extraction folders of files. - * - * @return A number greater than zero. - */ - @Override - public int depth() { - return this.input.getDepth(); + if (this.pipeline.getShotQueueSize() == ExtractionPipelineConfig.DEFAULT_SEGMENTQUEUE_SIZE) { + this.pipeline.setShotQueueSize(globalExt.getShotQueueSize()); } - /** - * Returns an instance of ObjectIdGenerator that should be used to generated MultimediaObject ID's - * during an extraction run. - * - * @return ObjectIdGenerator - */ - @Override - public ObjectIdGenerator objectIdGenerator() { - return this.getInput().getId().getGenerator(); + /* Set SegmenterConfig. */ + if (segmenter == null) { + segmenter = new SegmenterConfig(this.type); } + this.segmenter = segmenter; - @Override - public IdConfig.ExistenceCheck existenceCheck() { - return this.input.getId().getExistenceCheckMode(); + /* Set ImageCacheConfig. */ + if (cacheConfig == null) { + cacheConfig = Config.sharedConfig().getCache(); } - - /** - * Returns the PersistencyWriterSupplier that can be used during the extraction run to - * obtain PersistencyWriter instance. - * - * @return PersistencyWriterSupplier instance used obtain a PersistencyWriter. - */ - @Override - public PersistencyWriterSupplier persistencyWriter() { - return this.database.getWriterSupplier(); + this.cacheConfig = cacheConfig; + } + + @JsonProperty(required = true) + public InputConfig getInput() { + return input; + } + + @JsonProperty + public List getExtractors() { + return extractors; + } + + @JsonProperty + public List getExporters() { + return exporters; + } + + @JsonProperty + public List getMetadata() { + return metadata; + } + + @JsonProperty + public DatabaseConfig getDatabase() { + return database; + } + + @JsonProperty + public ExtractionPipelineConfig getPipeline() { + return pipeline; + } + + @JsonProperty + public SegmenterConfig getSegmenter() { + return this.segmenter; + } + + @Override + public Optional inputPath() { + if (this.input != null) { + return Optional.of(Paths.get(this.input.getPath())); + } else { + return Optional.empty(); } + } - /** - * Returns the size of a batch. A batch is used when persisting data. Entities will be kept in - * memory until the batchsize limit is hit at which point they will be persisted. - * - * @return Batch size. - */ - @Override - public Integer batchSize() { - return this.database.getBatchsize(); + @Override + public Optional relPath() { + if (this.input == null || this.input.getRelTo() == null) { + return Optional.empty(); } - - - /** - * Returns the DBSelectorSupplier that can be used during the extraction run to obtain - * a DBSelector instance. - * - * @return DBSelectorSupplier instance used obtain a DBSelector. - */ - @Override - public DBSelectorSupplier persistencyReader() { - return this.database.getSelectorSupplier(); - } - - @Override - public File outputLocation() { - return this.pipeline.getOutputLocation(); - } - - @Override - public int threadPoolSize() { - return this.pipeline.getThreadPoolSize(); - } - - /** - * Returns the size of the extraction task queue. Limits how many extraction tasks can be dispatched and kept in memory. - * - * @return Size of extraction task queue. - */ - @Override - public Integer taskQueueSize() { - return this.pipeline.getTaskQueueSize(); - } - - /** - * Returns the size of the segment queue. Segments can be created and kept in memory until that queue is full. - * - * @return Size of segment queue. - */ - @Override - public Integer segmentQueueSize() { - return this.pipeline.getShotQueueSize(); - } - - /** - * Returns the instance of {@link CacheConfig}. - * - * @return {@link CacheConfig} reference. - */ - @Override - public CacheConfig cacheConfig() { - return this.cacheConfig; + return Optional.of(Paths.get(this.input.getRelTo())); + } + + public ExtractionContainerProvider pathProvider() { + if (this.input != null) { + return new SingletonContainerProvider(Paths.get(this.input.getPath())); + } else { + return new SessionContainerProvider(); } + } + + /** + * Determines the MediaType of the source material. Only one media-type can be specified per ExtractionContextProvider. + * + * @return Media-type of the source material. + */ + @Override + @JsonProperty + public MediaType getType() { + return this.type; + } + + /** + * Returns a list of extractor classes that should be used for the extraction run! + * + * @return List of named extractors. + */ + @Override + public List extractors() { + return this.extractors.stream() + .map(ExtractorConfig::getExtractor) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Returns a list of exporter classes that should be invoked during extraction. Exporters usually generate some representation and persistently store that information somewhere. + * + * @return List of named exporters. + */ + @Override + public List exporters() { + return this.exporters.stream() + .map(ExtractorConfig::getExporter) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Returns a list of metadata extractor classes that should be invoked during extraction. MetadataExtractor's usually read some metadata from a file. + * + * @return List of named exporters. + */ + @Override + public List metadataExtractors() { + return this.metadata.stream() + .map(MetadataConfig::getMetadataExtractor) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Selects, configures and returns a new instance of the {@link Segmenter} that was configured in the current instance of {@link ExtractionContextProvider}. + * + * @return {@link Segmenter} that was configured in the current instance of {@link ExtractionContextProvider} + */ + @Override + public Segmenter newSegmenter() { + return this.segmenter.newSegmenter(this); + } + + /** + * Limits the depth of recursion when extraction folders of files. + * + * @return A number greater than zero. + */ + @Override + public int depth() { + return this.input.getDepth(); + } + + /** + * Returns an instance of ObjectIdGenerator that should be used to generated MultimediaObject ID's during an extraction run. + * + * @return ObjectIdGenerator + */ + @Override + public ObjectIdGenerator objectIdGenerator() { + return this.getInput().getId().getGenerator(); + } + + @Override + public IdConfig.ExistenceCheck existenceCheck() { + return this.input.getId().getExistenceCheckMode(); + } + + /** + * Returns the PersistencyWriterSupplier that can be used during the extraction run to obtain PersistencyWriter instance. + * + * @return PersistencyWriterSupplier instance used obtain a PersistencyWriter. + */ + @Override + public PersistencyWriterSupplier persistencyWriter() { + return this.database.getWriterSupplier(); + } + + /** + * Returns the size of a batch. A batch is used when persisting data. Entities will be kept in memory until the batchsize limit is hit at which point they will be persisted. + * + * @return Batch size. + */ + @Override + public Integer batchSize() { + return this.database.getBatchsize(); + } + + + /** + * Returns the DBSelectorSupplier that can be used during the extraction run to obtain a DBSelector instance. + * + * @return DBSelectorSupplier instance used obtain a DBSelector. + */ + @Override + public DBSelectorSupplier persistencyReader() { + return this.database.getSelectorSupplier(); + } + + @Override + public File outputLocation() { + return this.pipeline.getOutputLocation(); + } + + @Override + public int threadPoolSize() { + return this.pipeline.getThreadPoolSize(); + } + + /** + * Returns the size of the extraction task queue. Limits how many extraction tasks can be dispatched and kept in memory. + * + * @return Size of extraction task queue. + */ + @Override + public Integer taskQueueSize() { + return this.pipeline.getTaskQueueSize(); + } + + /** + * Returns the size of the segment queue. Segments can be created and kept in memory until that queue is full. + * + * @return Size of segment queue. + */ + @Override + public Integer segmentQueueSize() { + return this.pipeline.getShotQueueSize(); + } + + /** + * Returns the instance of {@link CacheConfig}. + * + * @return {@link CacheConfig} reference. + */ + @Override + public CacheConfig cacheConfig() { + return this.cacheConfig; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java index 71ab3cbd4..370e23592 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java @@ -2,44 +2,48 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.HashMap; - public class MetadataConfig { - /** Name of the MetadataExtractor. Must correspond to the simple-name or the FQN of the respective class. - * - * @see org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor - */ - private String name; - - /** Properties that are being used to initialize the Extractor. - * - * @see org.vitrivr.cineast.core.features.extractor.Extractor - */ - private HashMap properties = new HashMap<>(); - - @JsonProperty(required = true) - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } - - @JsonProperty - public HashMap getProperties() { - return properties; - } - public void setProperties(HashMap properties) { - this.properties = properties; - } - - @JsonIgnore - public MetadataExtractor getMetadataExtractor() { - MetadataExtractor extractor = ReflectionHelper.newMetadataExtractor(this.name); - return extractor; - } + + /** + * Name of the MetadataExtractor. Must correspond to the simple-name or the FQN of the respective class. + * + * @see org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor + */ + private String name; + + /** + * Properties that are being used to initialize the Extractor. + * + * @see org.vitrivr.cineast.core.features.extractor.Extractor + */ + private HashMap properties = new HashMap<>(); + + @JsonProperty(required = true) + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @JsonProperty + public HashMap getProperties() { + return properties; + } + + public void setProperties(HashMap properties) { + this.properties = properties; + } + + @JsonIgnore + public MetadataExtractor getMetadataExtractor() { + MetadataExtractor extractor = ReflectionHelper.newMetadataExtractor(this.name); + return extractor; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MonitoringConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MonitoringConfig.java index 590b367dc..a5ee8062c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MonitoringConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MonitoringConfig.java @@ -19,12 +19,14 @@ public void setPrometheusPort(int prometheusPort) { } @JsonCreator - public MonitoringConfig(){} + public MonitoringConfig() { + } @JsonProperty public boolean getEnablePrometheus() { return enablePrometheus; } + public void setEnablePrometheus(boolean enablePrometheus) { this.enablePrometheus = enablePrometheus; } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java index 60d00a36b..a05fe5ffb 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java @@ -3,151 +3,194 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import gnu.trove.map.hash.TObjectDoubleHashMap; -import org.vitrivr.cineast.core.features.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.vitrivr.cineast.core.features.AverageColor; +import org.vitrivr.cineast.core.features.AverageColorARP44; +import org.vitrivr.cineast.core.features.AverageColorCLD; +import org.vitrivr.cineast.core.features.AverageColorGrid8; +import org.vitrivr.cineast.core.features.AverageColorRaster; +import org.vitrivr.cineast.core.features.AverageFuzzyHist; +import org.vitrivr.cineast.core.features.CLD; +import org.vitrivr.cineast.core.features.ChromaGrid8; +import org.vitrivr.cineast.core.features.DescriptionTextSearch; +import org.vitrivr.cineast.core.features.DominantColors; +import org.vitrivr.cineast.core.features.DominantEdgeGrid16; +import org.vitrivr.cineast.core.features.DominantEdgeGrid8; +import org.vitrivr.cineast.core.features.EHD; +import org.vitrivr.cineast.core.features.EdgeARP88; +import org.vitrivr.cineast.core.features.EdgeARP88Full; +import org.vitrivr.cineast.core.features.EdgeGrid16; +import org.vitrivr.cineast.core.features.EdgeGrid16Full; +import org.vitrivr.cineast.core.features.HueValueVarianceGrid8; +import org.vitrivr.cineast.core.features.MedianColor; +import org.vitrivr.cineast.core.features.MedianColorARP44; +import org.vitrivr.cineast.core.features.MedianColorGrid8; +import org.vitrivr.cineast.core.features.MedianColorRaster; +import org.vitrivr.cineast.core.features.MedianFuzzyHist; +import org.vitrivr.cineast.core.features.MotionHistogram; +import org.vitrivr.cineast.core.features.SaturationGrid8; +import org.vitrivr.cineast.core.features.SubDivAverageFuzzyColor; +import org.vitrivr.cineast.core.features.SubDivMedianFuzzyColor; +import org.vitrivr.cineast.core.features.SubDivMotionHistogram2; +import org.vitrivr.cineast.core.features.SubDivMotionHistogram3; +import org.vitrivr.cineast.core.features.SubDivMotionHistogram4; +import org.vitrivr.cineast.core.features.SubDivMotionHistogram5; +import org.vitrivr.cineast.core.features.SubDivMotionSum2; +import org.vitrivr.cineast.core.features.SubDivMotionSum3; +import org.vitrivr.cineast.core.features.SubDivMotionSum4; +import org.vitrivr.cineast.core.features.SubDivMotionSum5; +import org.vitrivr.cineast.core.features.SubtitleFulltextSearch; import org.vitrivr.cineast.core.features.exporter.QueryImageExporter; import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.*; - public final class RetrievalRuntimeConfig { - private static final HashMap> DEFAULT_RETRIEVER_CATEGORIES = new HashMap<>(); - - private int threadPoolSize = 4; - private int taskQueueSize = 10; - private int maxResults = 100; - private int resultsPerModule = 50; - private HashMap> retrieverCategories = DEFAULT_RETRIEVER_CATEGORIES; - - static{ - - List list; - - list = new ArrayList<>(6); - list.add(new RetrieverConfig(AverageColor.class, 2.3)); - list.add(new RetrieverConfig(DominantColors.class, 1.0)); - list.add(new RetrieverConfig(MedianColor.class, 1.2)); - list.add(new RetrieverConfig(QueryImageExporter.class, 0.0001)); - list.add(new RetrieverConfig(AverageFuzzyHist.class, 0.7)); - list.add(new RetrieverConfig(MedianFuzzyHist.class, 1.3)); - DEFAULT_RETRIEVER_CATEGORIES.put("globalcolor", list); - - list = new ArrayList<>(13); - list.add(new RetrieverConfig(AverageColorARP44.class, 0.5)); - list.add(new RetrieverConfig(MedianColorARP44.class, 0.85)); - list.add(new RetrieverConfig(SubDivAverageFuzzyColor.class, 0.5)); - list.add(new RetrieverConfig(SubDivMedianFuzzyColor.class, 0.85)); - list.add(new RetrieverConfig(AverageColorGrid8.class, 1.8)); - list.add(new RetrieverConfig(ChromaGrid8.class, 0.95)); - list.add(new RetrieverConfig(SaturationGrid8.class, 0.65)); - list.add(new RetrieverConfig(AverageColorCLD.class, 1.4)); - list.add(new RetrieverConfig(CLD.class, 1.3)); - list.add(new RetrieverConfig(HueValueVarianceGrid8.class, 0.85)); - list.add(new RetrieverConfig(MedianColorGrid8.class, 1.7)); - list.add(new RetrieverConfig(AverageColorRaster.class, 1.0)); - list.add(new RetrieverConfig(MedianColorRaster.class, 1.0)); - DEFAULT_RETRIEVER_CATEGORIES.put("localcolor", list); - - list = new ArrayList<>(7); - list.add(new RetrieverConfig(EdgeARP88.class, 0.85)); - list.add(new RetrieverConfig(EdgeGrid16.class, 1.15)); - list.add(new RetrieverConfig(EdgeARP88Full.class, 0.85)); - list.add(new RetrieverConfig(EdgeGrid16Full.class, 0.85)); - list.add(new RetrieverConfig(EHD.class, 0.7)); - list.add(new RetrieverConfig(DominantEdgeGrid16.class, 1.4)); - list.add(new RetrieverConfig(DominantEdgeGrid8.class, 1.4)); - DEFAULT_RETRIEVER_CATEGORIES.put("edge", list); - - list = new ArrayList<>(9); - list.add(new RetrieverConfig(MotionHistogram.class, 0.5)); - list.add(new RetrieverConfig(SubDivMotionHistogram2.class, 1.0)); - list.add(new RetrieverConfig(SubDivMotionHistogram3.class, 1.0)); - list.add(new RetrieverConfig(SubDivMotionHistogram4.class, 1.0)); - list.add(new RetrieverConfig(SubDivMotionHistogram5.class, 1.0)); - list.add(new RetrieverConfig(SubDivMotionSum2.class, 0.5)); - list.add(new RetrieverConfig(SubDivMotionSum3.class, 0.5)); - list.add(new RetrieverConfig(SubDivMotionSum4.class, 0.5)); - list.add(new RetrieverConfig(SubDivMotionSum5.class, 0.5)); - DEFAULT_RETRIEVER_CATEGORIES.put("motion", list); - - list = new ArrayList<>(3); - list.add(new RetrieverConfig(SubtitleFulltextSearch.class, 1.0)); - list.add(new RetrieverConfig(DescriptionTextSearch.class, 1.0)); - + + private static final HashMap> DEFAULT_RETRIEVER_CATEGORIES = new HashMap<>(); + + private int threadPoolSize = 4; + private int taskQueueSize = 10; + private int maxResults = 100; + private int resultsPerModule = 50; + private HashMap> retrieverCategories = DEFAULT_RETRIEVER_CATEGORIES; + + static { + + List list; + + list = new ArrayList<>(6); + list.add(new RetrieverConfig(AverageColor.class, 2.3)); + list.add(new RetrieverConfig(DominantColors.class, 1.0)); + list.add(new RetrieverConfig(MedianColor.class, 1.2)); + list.add(new RetrieverConfig(QueryImageExporter.class, 0.0001)); + list.add(new RetrieverConfig(AverageFuzzyHist.class, 0.7)); + list.add(new RetrieverConfig(MedianFuzzyHist.class, 1.3)); + DEFAULT_RETRIEVER_CATEGORIES.put("globalcolor", list); + + list = new ArrayList<>(13); + list.add(new RetrieverConfig(AverageColorARP44.class, 0.5)); + list.add(new RetrieverConfig(MedianColorARP44.class, 0.85)); + list.add(new RetrieverConfig(SubDivAverageFuzzyColor.class, 0.5)); + list.add(new RetrieverConfig(SubDivMedianFuzzyColor.class, 0.85)); + list.add(new RetrieverConfig(AverageColorGrid8.class, 1.8)); + list.add(new RetrieverConfig(ChromaGrid8.class, 0.95)); + list.add(new RetrieverConfig(SaturationGrid8.class, 0.65)); + list.add(new RetrieverConfig(AverageColorCLD.class, 1.4)); + list.add(new RetrieverConfig(CLD.class, 1.3)); + list.add(new RetrieverConfig(HueValueVarianceGrid8.class, 0.85)); + list.add(new RetrieverConfig(MedianColorGrid8.class, 1.7)); + list.add(new RetrieverConfig(AverageColorRaster.class, 1.0)); + list.add(new RetrieverConfig(MedianColorRaster.class, 1.0)); + DEFAULT_RETRIEVER_CATEGORIES.put("localcolor", list); + + list = new ArrayList<>(7); + list.add(new RetrieverConfig(EdgeARP88.class, 0.85)); + list.add(new RetrieverConfig(EdgeGrid16.class, 1.15)); + list.add(new RetrieverConfig(EdgeARP88Full.class, 0.85)); + list.add(new RetrieverConfig(EdgeGrid16Full.class, 0.85)); + list.add(new RetrieverConfig(EHD.class, 0.7)); + list.add(new RetrieverConfig(DominantEdgeGrid16.class, 1.4)); + list.add(new RetrieverConfig(DominantEdgeGrid8.class, 1.4)); + DEFAULT_RETRIEVER_CATEGORIES.put("edge", list); + + list = new ArrayList<>(9); + list.add(new RetrieverConfig(MotionHistogram.class, 0.5)); + list.add(new RetrieverConfig(SubDivMotionHistogram2.class, 1.0)); + list.add(new RetrieverConfig(SubDivMotionHistogram3.class, 1.0)); + list.add(new RetrieverConfig(SubDivMotionHistogram4.class, 1.0)); + list.add(new RetrieverConfig(SubDivMotionHistogram5.class, 1.0)); + list.add(new RetrieverConfig(SubDivMotionSum2.class, 0.5)); + list.add(new RetrieverConfig(SubDivMotionSum3.class, 0.5)); + list.add(new RetrieverConfig(SubDivMotionSum4.class, 0.5)); + list.add(new RetrieverConfig(SubDivMotionSum5.class, 0.5)); + DEFAULT_RETRIEVER_CATEGORIES.put("motion", list); + + list = new ArrayList<>(3); + list.add(new RetrieverConfig(SubtitleFulltextSearch.class, 1.0)); + list.add(new RetrieverConfig(DescriptionTextSearch.class, 1.0)); + // list.add(new RetrieverConfig(QueryImageExporter.class, 0.001)); - DEFAULT_RETRIEVER_CATEGORIES.put("meta", list); - } - - @JsonCreator - public RetrievalRuntimeConfig() { - - } - - @JsonProperty - public int getThreadPoolSize(){ - return this.threadPoolSize; - } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } - - @JsonProperty - public int getTaskQueueSize() { - return this.taskQueueSize; - } - public void setTaskQueueSize(int taskQueueSize) { - this.taskQueueSize = taskQueueSize; - } - - @JsonProperty - public int getMaxResults(){ - return this.maxResults; - } - public void setMaxResults(int maxResults) { - this.maxResults = maxResults; - } - - @JsonProperty - public int getMaxResultsPerModule(){ - return this.resultsPerModule; - } - public void setResultsPerModule(int resultsPerModule) { - this.resultsPerModule = resultsPerModule; - } - - @JsonProperty("features") - public List getRetrieverCategories(){ - Set keys = this.retrieverCategories.keySet(); - ArrayList _return = new ArrayList<>(keys.size()); - _return.addAll(keys); - return _return; - } - - - public TObjectDoubleHashMap getRetrieversByCategory(String category){ - List list = this.retrieverCategories.get(category); - if(list == null){ - return new TObjectDoubleHashMap<>(1); - } - - TObjectDoubleHashMap _return = new TObjectDoubleHashMap<>(list.size()); - for(RetrieverConfig config : list){ - - Retriever rev; - - if(config.getProperties() == null){ - rev = ReflectionHelper.instantiate(config.getRetrieverClass()); - } else { - rev = ReflectionHelper.instantiate(config.getRetrieverClass(), config.getProperties()); - } - - if(rev != null){ - _return.put(rev, config.getWeight()); - } - } - - return _return; - } + DEFAULT_RETRIEVER_CATEGORIES.put("meta", list); + } + + @JsonCreator + public RetrievalRuntimeConfig() { + + } + + @JsonProperty + public int getThreadPoolSize() { + return this.threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + @JsonProperty + public int getTaskQueueSize() { + return this.taskQueueSize; + } + + public void setTaskQueueSize(int taskQueueSize) { + this.taskQueueSize = taskQueueSize; + } + + @JsonProperty + public int getMaxResults() { + return this.maxResults; + } + + public void setMaxResults(int maxResults) { + this.maxResults = maxResults; + } + + @JsonProperty + public int getMaxResultsPerModule() { + return this.resultsPerModule; + } + + public void setResultsPerModule(int resultsPerModule) { + this.resultsPerModule = resultsPerModule; + } + + @JsonProperty("features") + public List getRetrieverCategories() { + Set keys = this.retrieverCategories.keySet(); + ArrayList _return = new ArrayList<>(keys.size()); + _return.addAll(keys); + return _return; + } + + + public TObjectDoubleHashMap getRetrieversByCategory(String category) { + List list = this.retrieverCategories.get(category); + if (list == null) { + return new TObjectDoubleHashMap<>(1); + } + + TObjectDoubleHashMap _return = new TObjectDoubleHashMap<>(list.size()); + for (RetrieverConfig config : list) { + + Retriever rev; + + if (config.getProperties() == null) { + rev = ReflectionHelper.instantiate(config.getRetrieverClass()); + } else { + rev = ReflectionHelper.instantiate(config.getRetrieverClass(), config.getProperties()); + } + + if (rev != null) { + _return.put(rev, config.getWeight()); + } + } + + return _return; + } public Optional getRetrieverByName(String retrieverName) { for (List configs : this.retrieverCategories @@ -155,13 +198,13 @@ public Optional getRetrieverByName(String retrieverName) { for (RetrieverConfig config : configs) { if (config.getRetrieverClass().getSimpleName().equals(retrieverName)) { - Retriever retriever; + Retriever retriever; - if(config.getProperties() == null){ - retriever = ReflectionHelper.instantiate(config.getRetrieverClass()); - } else { - retriever = ReflectionHelper.instantiate(config.getRetrieverClass(), config.getProperties()); - } + if (config.getProperties() == null) { + retriever = ReflectionHelper.instantiate(config.getRetrieverClass()); + } else { + retriever = ReflectionHelper.instantiate(config.getRetrieverClass(), config.getProperties()); + } if (retriever != null) { return Optional.of(retriever); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java index 0e8444fd9..14903641d 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java @@ -2,52 +2,51 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.core.util.ReflectionHelper; -import java.util.Map; - public class RetrieverConfig { - private final Class retrieverClass; - private final double weight; - private final Map properties; - - RetrieverConfig(Class retrieverClass, double weight, Map properties){ - this.retrieverClass = retrieverClass; - this.weight = weight; - this.properties = properties; - } - - @JsonCreator - public RetrieverConfig( - @JsonProperty(value = "feature", required = true) String retrieverClassName, - @JsonProperty(value = "weight", required = false, defaultValue = "1.0") Double weight, - @JsonProperty(value = "properties", required = false) Map properties - ) throws InstantiationException, ClassNotFoundException { - this.retrieverClass = ReflectionHelper.getClassFromName(retrieverClassName, Retriever.class, ReflectionHelper.FEATURE_MODULE_PACKAGE); - this.weight = weight; - this.properties = properties; - } - - public RetrieverConfig(Class retrieverClass, double weight){ - this(retrieverClass, weight, null); - } - - public RetrieverConfig(Class retrieverClass){ - this(retrieverClass, 1.0); - } - - public Class getRetrieverClass(){ - return this.retrieverClass; - } - - public double getWeight(){ - return this.weight; - } - - public Map getProperties() { - return this.properties; - } + private final Class retrieverClass; + private final double weight; + private final Map properties; + + RetrieverConfig(Class retrieverClass, double weight, Map properties) { + this.retrieverClass = retrieverClass; + this.weight = weight; + this.properties = properties; + } + + @JsonCreator + public RetrieverConfig( + @JsonProperty(value = "feature", required = true) String retrieverClassName, + @JsonProperty(value = "weight", required = false, defaultValue = "1.0") Double weight, + @JsonProperty(value = "properties", required = false) Map properties + ) throws InstantiationException, ClassNotFoundException { + this.retrieverClass = ReflectionHelper.getClassFromName(retrieverClassName, Retriever.class, ReflectionHelper.FEATURE_MODULE_PACKAGE); + this.weight = weight; + this.properties = properties; + } + + public RetrieverConfig(Class retrieverClass, double weight) { + this(retrieverClass, weight, null); + } + + public RetrieverConfig(Class retrieverClass) { + this(retrieverClass, 1.0); + } + + public Class getRetrieverClass() { + return this.retrieverClass; + } + + public double getWeight() { + return this.weight; + } + + public Map getProperties() { + return this.properties; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationConfig.java index 71075e096..d98a44cbe 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationConfig.java @@ -1,111 +1,134 @@ package org.vitrivr.cineast.standalone.evaluation; import com.fasterxml.jackson.annotation.JsonProperty; -import org.vitrivr.cineast.core.extraction.decode.general.Converter; -import org.vitrivr.cineast.core.util.ReflectionHelper; - import java.nio.file.Path; import java.nio.file.Paths; +import org.vitrivr.cineast.core.extraction.decode.general.Converter; +import org.vitrivr.cineast.core.util.ReflectionHelper; public class EvaluationConfig { - /** Evaluation mode determines how files should be selected; - * - COMPLETE: All files in the test files folder. - * - RANDOM: Random selection of files from the test files folder. - */ - public enum EvaluationMode { - COMPLETE,RANDOM - } - - /** Path to the folder containing the test files. */ - private String testfiles; - - /** Path to the file hat describes the ground truth. */ - private String classfile; - - /** Path to the folder where results should be written. */ - private String results; - - /** The fully qualified name of the converter class used to convert test files to QueryContainers. */ - private String converter; - - /** Delimiter used between columns in the text export. */ - private String delimiter = ","; - - /** The query categories that should be tested. */ - private String[] categories = new String[0]; - - /** The evaluation mode that should be used. */ - private EvaluationMode mode = EvaluationMode.COMPLETE; - - /** The size of the test set (i.e. the number of elements in the database). */ - private int size = 0; - - @JsonProperty - public Path getTestfiles() { - return Paths.get(testfiles); - } - public void setTestfiles(String testfiles) { - this.testfiles = testfiles; - } - - @JsonProperty - public Path getClassfile() { - return Paths.get(this.classfile); - } - public void setClassfile(String classfile) { - this.classfile = classfile; - } - public Groundtruth getGroundtruth() throws EvaluationException { - return new Groundtruth(this.getClassfile()); - } - - @JsonProperty - public Path getResults() { - return Paths.get(this.results); - } - public void setResults(String results) { - this.results = results; - } - - @JsonProperty - public Converter getConverter() { - return ReflectionHelper.newConverter(this.converter); - } - public void setConverter(String converter) { - this.converter = converter; - } - - @JsonProperty - public String[] getCategories() { - return categories; - } - public void setCategories(String[] categories) { - this.categories = categories; - } - - @JsonProperty - public int getSize() { - return size; - } - public void setSize(int size) { - this.size = size; - } - - @JsonProperty - public String getDelimiter() { - return delimiter; - } - public void setDelimiter(String delimiter) { - this.delimiter = delimiter; - } - - @JsonProperty - public EvaluationMode getMode() { - return mode; - } - public void setMode(EvaluationMode mode) { - this.mode = mode; - } + /** + * Evaluation mode determines how files should be selected; - COMPLETE: All files in the test files folder. - RANDOM: Random selection of files from the test files folder. + */ + public enum EvaluationMode { + COMPLETE, RANDOM + } + + /** + * Path to the folder containing the test files. + */ + private String testfiles; + + /** + * Path to the file hat describes the ground truth. + */ + private String classfile; + + /** + * Path to the folder where results should be written. + */ + private String results; + + /** + * The fully qualified name of the converter class used to convert test files to QueryContainers. + */ + private String converter; + + /** + * Delimiter used between columns in the text export. + */ + private String delimiter = ","; + + /** + * The query categories that should be tested. + */ + private String[] categories = new String[0]; + + /** + * The evaluation mode that should be used. + */ + private EvaluationMode mode = EvaluationMode.COMPLETE; + + /** + * The size of the test set (i.e. the number of elements in the database). + */ + private int size = 0; + + @JsonProperty + public Path getTestfiles() { + return Paths.get(testfiles); + } + + public void setTestfiles(String testfiles) { + this.testfiles = testfiles; + } + + @JsonProperty + public Path getClassfile() { + return Paths.get(this.classfile); + } + + public void setClassfile(String classfile) { + this.classfile = classfile; + } + + public Groundtruth getGroundtruth() throws EvaluationException { + return new Groundtruth(this.getClassfile()); + } + + @JsonProperty + public Path getResults() { + return Paths.get(this.results); + } + + public void setResults(String results) { + this.results = results; + } + + @JsonProperty + public Converter getConverter() { + return ReflectionHelper.newConverter(this.converter); + } + + public void setConverter(String converter) { + this.converter = converter; + } + + @JsonProperty + public String[] getCategories() { + return categories; + } + + public void setCategories(String[] categories) { + this.categories = categories; + } + + @JsonProperty + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + @JsonProperty + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + @JsonProperty + public EvaluationMode getMode() { + return mode; + } + + public void setMode(EvaluationMode mode) { + this.mode = mode; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationException.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationException.java index 683c38aca..aaace1242 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationException.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationException.java @@ -2,18 +2,17 @@ /** * An exception that indicates something went wrong during an Evaluation-Run. - * */ public class EvaluationException extends Exception { - static final long serialVersionUID = 1L; + static final long serialVersionUID = 1L; - /** - * Constructor for EvaluationException - * - * @param message Error message. - */ - EvaluationException(String message) { - super(message); - } + /** + * Constructor for EvaluationException + * + * @param message Error message. + */ + EvaluationException(String message) { + super(message); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java index 68e2f9835..003136681 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationResult.java @@ -1,156 +1,164 @@ package org.vitrivr.cineast.standalone.evaluation; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; /** - * Helper class that can be used to track evaluation results. It tracks the total number of relevant documents - * per class, the number of retrieved documents and the number of hits at every rank k. - * + * Helper class that can be used to track evaluation results. It tracks the total number of relevant documents per class, the number of retrieved documents and the number of hits at every rank k. */ public class EvaluationResult { - /** ID of the reference document that was used to perform the query. */ - private final String docID; - - /** Name of reference object's class labels. */ - private final String cl; - - /** Number of relevant documents in the class according to the ground truth. */ - private final int relevant; - - /** Number of retrieved results. */ - private int retrieved = 0; - - /** Number of retrieved and relevant documents. */ - private int intersection = 0; - - /** List of docID / precision / recall triples for each rank. */ - private List> pk; - - /** - * Creates a new EvaluationResult object for the provided reference document - * - * @param docID ID of the reference document. - * @param groundtruth Ground truth object used to construct this evaluation result. - */ - public EvaluationResult(String docID, Groundtruth groundtruth) throws EvaluationException { - this.docID = docID; - this.cl = groundtruth.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided document ID '%s' is not registered in the ground truth.", docID))); - this.relevant = groundtruth.numberOfRelevant(this.cl); - this.pk = new ArrayList<>(); - } - - /** - * Registers a new document from the resultset alongside with the information whether it was a hit or not. - * Updates the retrieval statistics. - * - * @param docID ID of the document that was retrieved. - * @param k The rank of the retrieved document. - * @param relevant Boolean that indicates whether the document was relevant (true) or not (false). - */ - public final void documentAvailable(String docID, int k, boolean relevant) { - if (k < 1) { - throw new IllegalArgumentException(String.format("The value k must be greater than 0 (is: %d).", k)); - } - if (k < this.pk.size()) { - throw new IllegalArgumentException(String.format("The provided rank %d has already been evaluated.", k)); - } - if (relevant) { - this.intersection += 1; - } - this.retrieved += 1; - Triple triple = new ImmutableTriple<>(docID, (float)this.intersection/(float)k, (float)this.intersection/(float)this.relevant); - this.pk.add(triple); - } - /** - * Getter for query object ID. - */ - public final String getDocId() { - return this.docID; + /** + * ID of the reference document that was used to perform the query. + */ + private final String docID; + + /** + * Name of reference object's class labels. + */ + private final String cl; + + /** + * Number of relevant documents in the class according to the ground truth. + */ + private final int relevant; + + /** + * Number of retrieved results. + */ + private int retrieved = 0; + + /** + * Number of retrieved and relevant documents. + */ + private int intersection = 0; + + /** + * List of docID / precision / recall triples for each rank. + */ + private List> pk; + + /** + * Creates a new EvaluationResult object for the provided reference document + * + * @param docID ID of the reference document. + * @param groundtruth Ground truth object used to construct this evaluation result. + */ + public EvaluationResult(String docID, Groundtruth groundtruth) throws EvaluationException { + this.docID = docID; + this.cl = groundtruth.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided document ID '%s' is not registered in the ground truth.", docID))); + this.relevant = groundtruth.numberOfRelevant(this.cl); + this.pk = new ArrayList<>(); + } + + /** + * Registers a new document from the resultset alongside with the information whether it was a hit or not. Updates the retrieval statistics. + * + * @param docID ID of the document that was retrieved. + * @param k The rank of the retrieved document. + * @param relevant Boolean that indicates whether the document was relevant (true) or not (false). + */ + public final void documentAvailable(String docID, int k, boolean relevant) { + if (k < 1) { + throw new IllegalArgumentException(String.format("The value k must be greater than 0 (is: %d).", k)); } - - /** - * Getter for query object class. - */ - public final String getCl() { - return cl; - } - - /** - * Getter for relevant. - * - * @return Number of relevant documents as per ground truth. - */ - public int getRelevant() { - return relevant; - } - - /** - * Getter for retrieved. - * - * @return Number of retrieved documents. - */ - public final int getRetrieved() { - return retrieved; + if (k < this.pk.size()) { + throw new IllegalArgumentException(String.format("The provided rank %d has already been evaluated.", k)); } - - /** - * Getter for intersection - * - * @return Number of retrieved & relevant documents. - */ - public final int getIntersection() { - return intersection; + if (relevant) { + this.intersection += 1; } - - /** - * Returns true, if the number of retrieved & relevant documents equals the - * total number of relevant documents. - */ - public boolean done() { - return this.intersection == this.relevant; + this.retrieved += 1; + Triple triple = new ImmutableTriple<>(docID, (float) this.intersection / (float) k, (float) this.intersection / (float) this.relevant); + this.pk.add(triple); + } + + /** + * Getter for query object ID. + */ + public final String getDocId() { + return this.docID; + } + + /** + * Getter for query object class. + */ + public final String getCl() { + return cl; + } + + /** + * Getter for relevant. + * + * @return Number of relevant documents as per ground truth. + */ + public int getRelevant() { + return relevant; + } + + /** + * Getter for retrieved. + * + * @return Number of retrieved documents. + */ + public final int getRetrieved() { + return retrieved; + } + + /** + * Getter for intersection + * + * @return Number of retrieved & relevant documents. + */ + public final int getIntersection() { + return intersection; + } + + /** + * Returns true, if the number of retrieved & relevant documents equals the total number of relevant documents. + */ + public boolean done() { + return this.intersection == this.relevant; + } + + public final String toString(String delimiter) { + StringBuilder builder = new StringBuilder(); + + /* Create header. */ + builder.append("ID"); + builder.append(delimiter); + builder.append("Hit"); + builder.append(delimiter); + builder.append("Precision"); + builder.append(delimiter); + builder.append("Recall"); + builder.append("\n"); + + int i = 0; + /* Append data. */ + for (Triple triple : this.pk) { + builder.append(triple.getLeft()); + builder.append(delimiter); + + /* Check if entry was a documentAvailable. */ + boolean hit = false; + if (i == 0 && triple.getRight() > 0) { + hit = true; + } else if (i > 0 && triple.getRight() > this.pk.get(i - 1).getRight()) { + hit = true; + } + builder.append(hit ? 1 : 0); + builder.append(delimiter); + + builder.append(triple.getMiddle()); + builder.append(delimiter); + builder.append(triple.getRight()); + builder.append("\n"); + i++; } - public final String toString(String delimiter) { - StringBuilder builder = new StringBuilder(); - - /* Create header. */ - builder.append("ID"); - builder.append(delimiter); - builder.append("Hit"); - builder.append(delimiter); - builder.append("Precision"); - builder.append(delimiter); - builder.append("Recall"); - builder.append("\n"); - - int i = 0; - /* Append data. */ - for (Triple triple : this.pk) { - builder.append(triple.getLeft()); - builder.append(delimiter); - - /* Check if entry was a documentAvailable. */ - boolean hit = false; - if (i == 0 && triple.getRight() > 0) { - hit = true; - } else if (i > 0 && triple.getRight() > this.pk.get(i-1).getRight()) { - hit = true; - } - builder.append(hit ? 1 : 0); - builder.append(delimiter); - - builder.append(triple.getMiddle()); - builder.append(delimiter); - builder.append(triple.getRight()); - builder.append("\n"); - i++; - } - - return builder.toString(); - } + return builder.toString(); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationRuntime.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationRuntime.java index 0e9c739df..9b4203ef5 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationRuntime.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/EvaluationRuntime.java @@ -1,5 +1,16 @@ package org.vitrivr.cineast.standalone.evaluation; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.Callable; import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; @@ -19,217 +30,216 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.util.ContinuousRetrievalLogic; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.*; -import java.util.concurrent.Callable; - public class EvaluationRuntime implements Callable { - private static final Logger LOGGER = LogManager.getLogger(); - - /** MediaSegmentReader instance used to read segments from the storage layer. */ - private final MediaSegmentReader mediaSegmentReader; - - /** MediaObjectReader instance used to read multimedia objects from the storage layer. */ - private final MediaObjectReader mediaObjectReader; - - /** Instance of EvaluationConfig that is used with this runtime. */ - private final EvaluationConfig config; - - /** Caching structure used to cache MultimediaObjectDescriptors. */ - private final HashMap cache; - - /** Number of files that were processed successfully. */ - private int processed = 0; - - /** Number of files that were skipped deliberately. */ - private int skipped = 0; - - /** Number of files that were skipped due to processing errors. */ - private int error = 0; - - private final ContinuousRetrievalLogic retrievalLogic; - - - public EvaluationRuntime(Path configPath, DatabaseConfig dbConfig) { - JsonReader reader = new JacksonJsonProvider(); - this.config = reader.toObject(configPath.toFile(), EvaluationConfig.class); - this.cache = new HashMap<>(this.config.getSize()); - this.mediaSegmentReader = new MediaSegmentReader(dbConfig.getSelectorSupplier().get()); - this.mediaObjectReader = new MediaObjectReader(dbConfig.getSelectorSupplier().get()); - this.retrievalLogic = new ContinuousRetrievalLogic(dbConfig); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * MediaSegmentReader instance used to read segments from the storage layer. + */ + private final MediaSegmentReader mediaSegmentReader; + + /** + * MediaObjectReader instance used to read multimedia objects from the storage layer. + */ + private final MediaObjectReader mediaObjectReader; + + /** + * Instance of EvaluationConfig that is used with this runtime. + */ + private final EvaluationConfig config; + + /** + * Caching structure used to cache MultimediaObjectDescriptors. + */ + private final HashMap cache; + + /** + * Number of files that were processed successfully. + */ + private int processed = 0; + + /** + * Number of files that were skipped deliberately. + */ + private int skipped = 0; + + /** + * Number of files that were skipped due to processing errors. + */ + private int error = 0; + + private final ContinuousRetrievalLogic retrievalLogic; + + + public EvaluationRuntime(Path configPath, DatabaseConfig dbConfig) { + JsonReader reader = new JacksonJsonProvider(); + this.config = reader.toObject(configPath.toFile(), EvaluationConfig.class); + this.cache = new HashMap<>(this.config.getSize()); + this.mediaSegmentReader = new MediaSegmentReader(dbConfig.getSelectorSupplier().get()); + this.mediaObjectReader = new MediaObjectReader(dbConfig.getSelectorSupplier().get()); + this.retrievalLogic = new ContinuousRetrievalLogic(dbConfig); + } + + /** + * Executes the evaluation and returns a Triple that contains the number of files that were processed, skipped due to errors and skipped deliberately. The actual evaluation results are written to files. + * + * @return computed result + * @throws EvaluationException if unable to compute a result + * @see EvaluationConfig + */ + @Override + public Triple call() throws EvaluationException, IOException { + /* Tries to instantiate the converter. */ + final Converter converter = this.config.getConverter(); + if (converter == null) { + throw new EvaluationException("Failed to instantiate the converter class."); } - /** - * Executes the evaluation and returns a Triple that contains the number of files that were processed, - * skipped due to errors and skipped deliberately. The actual evaluation results are written to files. - * - * @see EvaluationConfig - * - * @return computed result - * @throws EvaluationException if unable to compute a result - */ - @Override - public Triple call() throws EvaluationException, IOException { - /* Tries to instantiate the converter. */ - final Converter converter = this.config.getConverter(); - if (converter == null) { - throw new EvaluationException("Failed to instantiate the converter class."); - } - - /* Instantiates the groundtruth and checks if it contains classes. */ - final Groundtruth gt = this.config.getGroundtruth(); - if (gt.numberOfClasses() == 0) { - throw new EvaluationException(String.format("The specified ground truth '%s' does not contain any classes.", this.config.getClassfile())); - } + /* Instantiates the groundtruth and checks if it contains classes. */ + final Groundtruth gt = this.config.getGroundtruth(); + if (gt.numberOfClasses() == 0) { + throw new EvaluationException(String.format("The specified ground truth '%s' does not contain any classes.", this.config.getClassfile())); + } - /* Updates the retrieval configuration. */ - Config.sharedConfig().getRetriever().setMaxResults(this.config.getSize()); - Config.sharedConfig().getRetriever().setResultsPerModule(this.config.getSize()); + /* Updates the retrieval configuration. */ + Config.sharedConfig().getRetriever().setMaxResults(this.config.getSize()); + Config.sharedConfig().getRetriever().setResultsPerModule(this.config.getSize()); - /* Prepares the iterator for the test files. */ - final Iterator testfilesIterator; + /* Prepares the iterator for the test files. */ + final Iterator testfilesIterator; + try { + testfilesIterator = Files.walk(this.config.getTestfiles()).filter(p -> { try { - testfilesIterator = Files.walk(this.config.getTestfiles()).filter(p -> { - try { - return Files.exists(p) && Files.isRegularFile(p) && !Files.isHidden(p) && Files.isReadable(p); - } catch (IOException e) { - LOGGER.error("An IO exception occurred while testing the media file at '{}'.", p.toString(), LogHelper.getStackTrace(e)); - return false; - } - }).iterator(); - } catch (IOException exception) { - throw new EvaluationException(String.format("Could not obtain test files under the specified path '%s'.", this.config.getTestfiles())); + return Files.exists(p) && Files.isRegularFile(p) && !Files.isHidden(p) && Files.isReadable(p); + } catch (IOException e) { + LOGGER.error("An IO exception occurred while testing the media file at '{}'.", p.toString(), LogHelper.getStackTrace(e)); + return false; } - - /* Prepare folder structure per category. */ - for (String category : this.config.getCategories()) { - Files.createDirectories(this.config.getResults().resolve(category)); - } - - /* Prepare a placeholder query-config. */ - final ReadableQueryConfig queryConfig = new ReadableQueryConfig(null); - - /* Prepare a random number generator that decides if a file should be used for evaluation or not. */ - final Random random = new Random(); - - /* Perform evaluation for every file. */ - Path path; - while(testfilesIterator.hasNext()) { - path = testfilesIterator.next(); - if (random.nextBoolean() && config.getMode() == EvaluationConfig.EvaluationMode.RANDOM) { - LOGGER.info("Randomly skipping file {}.", path); - this.skipped += 1; - continue; - } - - /* Try to create a QueryContainer. If this fails, the file is skipped. */ - final AbstractQueryTermContainer container = converter.convert(path); - if (container == null) { - LOGGER.warn("Failed to convert the file {}. File is being skipped...", path.getFileName()); - this.error += 1; - continue; - } - - LOGGER.info("Starting evaluation for {}", path); - for (String category : this.config.getCategories()) { - List scores = this.retrievalLogic.retrieve(container, category, queryConfig); - EvaluationResult result = this.performEvaluation(scores, path, gt); - this.writeToFile(category, result); - } - - this.processed += 1; - } - - return new ImmutableTriple<>(this.processed, this.error, this.skipped); + }).iterator(); + } catch (IOException exception) { + throw new EvaluationException(String.format("Could not obtain test files under the specified path '%s'.", this.config.getTestfiles())); } - - /** - * Writes the results for a feature category to a CSV file. If a file with that name already exists, the - * existing file is overwritten! - * - * @param category The feature category of the file. - * @param result The evaluation results. - * @throws IOException If writing of file fails. - */ - private void writeToFile(String category, EvaluationResult result) throws IOException { - BufferedWriter writer = Files.newBufferedWriter(this.config.getResults().resolve(category).resolve(result.getDocId() + "-" + result.getCl() + "-results.csv"), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - writer.append(result.toString(this.config.getDelimiter())); - writer.newLine(); - writer.close(); + /* Prepare folder structure per category. */ + for (String category : this.config.getCategories()) { + Files.createDirectories(this.config.getResults().resolve(category)); } - /** - * Performs the actual evaluation for a result set and reference document. Now iterate through the list of retrieved documents and - * decides for each entry if its class is equal to the class of the reference document (i.e. if it is relevant or not). Based on - * the outcome, the EvaluationResult is updated. - * - * @param scores The result set - list of SegmentScoreElements. - * @param path Path to the reference document file. Its filename is used as docID - * @param gt Ground truth object used for evaluation. - * @return EvaluationResult - * - * @throws EvaluationException If something goes wrong during evaluation, e.g. reference document has no class or a item in the result set has no class - */ - private EvaluationResult performEvaluation(List scores, Path path, Groundtruth gt) throws EvaluationException { - /* Constructs a document ID from the filename and fetches the file's class from the ground truth. */ - String docID = path.getFileName().toString(); - String fileClass = gt.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided test file %s does not have a class associated with it.",path.getFileName()))); - - /* Prepare empty evaluation results. */ - EvaluationResult result = new EvaluationResult(docID, gt); - - /* - * Now iterate through the list of retrieved documents and decide for each entry if it relevant according to its class. - */ - for (int k=1; k<=scores.size(); k++) { - SegmentScoreElement score = scores.get(k-1); - MediaObjectDescriptor object = this.objectDescriptorForId(score.getSegmentId()); - if (object != null) { - if (gt.classForDocId(object.getName()).orElse("").equals(fileClass)) { - result.documentAvailable(object.getName(), k, true); - } else { - result.documentAvailable(object.getName(), k, false); - } - if (result.done()) { - LOGGER.info("All relevant objects were retrieved. Starting next round...", score.getSegmentId()); - break; - } - } else { - throw new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", score.getSegmentId())); - } - } - - return result; + /* Prepare a placeholder query-config. */ + final ReadableQueryConfig queryConfig = new ReadableQueryConfig(null); + + /* Prepare a random number generator that decides if a file should be used for evaluation or not. */ + final Random random = new Random(); + + /* Perform evaluation for every file. */ + Path path; + while (testfilesIterator.hasNext()) { + path = testfilesIterator.next(); + if (random.nextBoolean() && config.getMode() == EvaluationConfig.EvaluationMode.RANDOM) { + LOGGER.info("Randomly skipping file {}.", path); + this.skipped += 1; + continue; + } + + /* Try to create a QueryContainer. If this fails, the file is skipped. */ + final AbstractQueryTermContainer container = converter.convert(path); + if (container == null) { + LOGGER.warn("Failed to convert the file {}. File is being skipped...", path.getFileName()); + this.error += 1; + continue; + } + + LOGGER.info("Starting evaluation for {}", path); + for (String category : this.config.getCategories()) { + List scores = this.retrievalLogic.retrieve(container, category, queryConfig); + EvaluationResult result = this.performEvaluation(scores, path, gt); + this.writeToFile(category, result); + } + + this.processed += 1; } - /** - * Returns a MediaObjectDescriptor for the provided docID. This method uses a cache to speedup - * lookup of objects. - * - * @param docID ID of the segment for which the MediaObjectDescriptor is required. - * @return MediaObjectDescriptor + return new ImmutableTriple<>(this.processed, this.error, this.skipped); + } + + + /** + * Writes the results for a feature category to a CSV file. If a file with that name already exists, the existing file is overwritten! + * + * @param category The feature category of the file. + * @param result The evaluation results. + * @throws IOException If writing of file fails. + */ + private void writeToFile(String category, EvaluationResult result) throws IOException { + BufferedWriter writer = Files.newBufferedWriter(this.config.getResults().resolve(category).resolve(result.getDocId() + "-" + result.getCl() + "-results.csv"), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + writer.append(result.toString(this.config.getDelimiter())); + writer.newLine(); + writer.close(); + } + + /** + * Performs the actual evaluation for a result set and reference document. Now iterate through the list of retrieved documents and decides for each entry if its class is equal to the class of the reference document (i.e. if it is relevant or not). Based on the outcome, the EvaluationResult is updated. + * + * @param scores The result set - list of SegmentScoreElements. + * @param path Path to the reference document file. Its filename is used as docID + * @param gt Ground truth object used for evaluation. + * @return EvaluationResult + * @throws EvaluationException If something goes wrong during evaluation, e.g. reference document has no class or a item in the result set has no class + */ + private EvaluationResult performEvaluation(List scores, Path path, Groundtruth gt) throws EvaluationException { + /* Constructs a document ID from the filename and fetches the file's class from the ground truth. */ + String docID = path.getFileName().toString(); + String fileClass = gt.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", path.getFileName()))); + + /* Prepare empty evaluation results. */ + EvaluationResult result = new EvaluationResult(docID, gt); + + /* + * Now iterate through the list of retrieved documents and decide for each entry if it relevant according to its class. */ - private MediaObjectDescriptor objectDescriptorForId(String docID) { - if (this.cache.containsKey(docID)) { - return this.cache.get(docID); - } - Optional descriptor = this.mediaSegmentReader.lookUpSegment(docID); - if (descriptor.isPresent()) { - MediaObjectDescriptor object = this.mediaObjectReader.lookUpObjectById(descriptor.get().getObjectId()); - this.cache.put(docID, object); - return object; + for (int k = 1; k <= scores.size(); k++) { + SegmentScoreElement score = scores.get(k - 1); + MediaObjectDescriptor object = this.objectDescriptorForId(score.getSegmentId()); + if (object != null) { + if (gt.classForDocId(object.getName()).orElse("").equals(fileClass)) { + result.documentAvailable(object.getName(), k, true); } else { - return null; + result.documentAvailable(object.getName(), k, false); + } + if (result.done()) { + LOGGER.info("All relevant objects were retrieved. Starting next round...", score.getSegmentId()); + break; } + } else { + throw new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", score.getSegmentId())); + } + } + + return result; + } + + /** + * Returns a MediaObjectDescriptor for the provided docID. This method uses a cache to speedup lookup of objects. + * + * @param docID ID of the segment for which the MediaObjectDescriptor is required. + * @return MediaObjectDescriptor + */ + private MediaObjectDescriptor objectDescriptorForId(String docID) { + if (this.cache.containsKey(docID)) { + return this.cache.get(docID); + } + Optional descriptor = this.mediaSegmentReader.lookUpSegment(docID); + if (descriptor.isPresent()) { + MediaObjectDescriptor object = this.mediaObjectReader.lookUpObjectById(descriptor.get().getObjectId()); + this.cache.put(docID, object); + return object; + } else { + return null; } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/Groundtruth.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/Groundtruth.java index 0b425b105..42980fdbe 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/Groundtruth.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/evaluation/Groundtruth.java @@ -2,130 +2,133 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.IOException; import java.nio.file.Path; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; /** - * Represents a ground truth data collection that assigns IDs of test objects to arbitrary classes and - * vice versa. This class can be constructed from a simple JSON file. - * + * Represents a ground truth data collection that assigns IDs of test objects to arbitrary classes and vice versa. This class can be constructed from a simple JSON file. */ public class Groundtruth { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - /** Maps class labels to document IDs. */ - private HashMap> map; - - /** Maps documentIDs class labels. */ - private HashMap inverseMap; - - /** - * Constructor for Groundtruth from a file. - * - * @param path Path to file from which to read the ground truth. - */ - public Groundtruth(Path path) throws EvaluationException { - this.map = new HashMap<>(); - this.inverseMap = new HashMap<>(); - try { - JsonNode node = MAPPER.readTree(path.toFile()); - if (node == null) { - throw new EvaluationException("Could not read the ground truth file from the specified path '%s'. File seems to be empty."); - } - this.readFromJsonNode(node); - } catch (IOException e) { - throw new EvaluationException(String.format("Could not read the ground truth file from the specified path '%s' due to an IOException.", path.toString())); - } - } - - /** - * Constructor for Groundtruth from a JsonNode. - * - * @param node JsonNode from which to read the ground truth. - */ - public Groundtruth (JsonNode node) throws EvaluationException { - this.map = new HashMap<>(); - this.inverseMap = new HashMap<>(); - this.readFromJsonNode(node); - } - - /** - * Returns the class label for the provided ID or an empty Optional, if no class - * has been defined for the provided ID. - * - * @return String class label for the provided ID. - */ - public final Optional classForDocId(String id) { - return Optional.ofNullable(this.inverseMap.get(id)); - } - - /** - * Returns the IDs saved under the provided class according to the ground truth. The return set - * is empty if the class does not exist. - * - * @return Set of IDs for the provided class. - */ - public final Set docIdsForClass(String c) { - return Collections.unmodifiableSet(this.map.getOrDefault(c, new HashSet<>(0))); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** + * Maps class labels to document IDs. + */ + private HashMap> map; + + /** + * Maps documentIDs class labels. + */ + private HashMap inverseMap; + + /** + * Constructor for Groundtruth from a file. + * + * @param path Path to file from which to read the ground truth. + */ + public Groundtruth(Path path) throws EvaluationException { + this.map = new HashMap<>(); + this.inverseMap = new HashMap<>(); + try { + JsonNode node = MAPPER.readTree(path.toFile()); + if (node == null) { + throw new EvaluationException("Could not read the ground truth file from the specified path '%s'. File seems to be empty."); + } + this.readFromJsonNode(node); + } catch (IOException e) { + throw new EvaluationException(String.format("Could not read the ground truth file from the specified path '%s' due to an IOException.", path.toString())); } - - /** - * Returns the number of classes registered in the Groundtruth. - * - * @return Number of classes. - */ - public final int numberOfClasses() { - return this.map.size(); + } + + /** + * Constructor for Groundtruth from a JsonNode. + * + * @param node JsonNode from which to read the ground truth. + */ + public Groundtruth(JsonNode node) throws EvaluationException { + this.map = new HashMap<>(); + this.inverseMap = new HashMap<>(); + this.readFromJsonNode(node); + } + + /** + * Returns the class label for the provided ID or an empty Optional, if no class has been defined for the provided ID. + * + * @return String class label for the provided ID. + */ + public final Optional classForDocId(String id) { + return Optional.ofNullable(this.inverseMap.get(id)); + } + + /** + * Returns the IDs saved under the provided class according to the ground truth. The return set is empty if the class does not exist. + * + * @return Set of IDs for the provided class. + */ + public final Set docIdsForClass(String c) { + return Collections.unmodifiableSet(this.map.getOrDefault(c, new HashSet<>(0))); + } + + /** + * Returns the number of classes registered in the Groundtruth. + * + * @return Number of classes. + */ + public final int numberOfClasses() { + return this.map.size(); + } + + + /** + * Returns the number of relevant documents for the provided class label. + * + * @param cl Class label. + * @return Number of relevant documents in that class. + */ + public final int numberOfRelevant(String cl) { + if (this.map.containsKey(cl)) { + return this.map.get(cl).size(); + } else { + return 0; } - - - /** - * Returns the number of relevant documents for the provided class - * label. - * - * @param cl Class label. - * @return Number of relevant documents in that class. - */ - public final int numberOfRelevant(String cl) { - if (this.map.containsKey(cl)) { - return this.map.get(cl).size(); - } else { - return 0; - } - } - - /** - * This method reads ground truth data from a JSON file. - * - * @param node JsonNode from which to read the ground truth. - */ - private void readFromJsonNode(JsonNode node) throws EvaluationException { - Iterator classes = node.fieldNames(); - while (classes.hasNext()) { - String cl = classes.next(); - JsonNode ids = node.get(cl); - - if (ids == null) { - continue; - } - - /* Add class label to map if it's not already there. */ - if (!this.map.containsKey(cl)) { - this.map.put(cl, new HashSet<>()); - } - - if (ids.isArray()) { - for (JsonNode item : ids) { - if (this.inverseMap.containsKey(cl)) { - throw new EvaluationException("Could not create ground truth due to a structural error: An ID must not belong to more than one class."); - } - this.map.get(cl).add(item.textValue()); - this.inverseMap.put(item.textValue(), cl); - } - } + } + + /** + * This method reads ground truth data from a JSON file. + * + * @param node JsonNode from which to read the ground truth. + */ + private void readFromJsonNode(JsonNode node) throws EvaluationException { + Iterator classes = node.fieldNames(); + while (classes.hasNext()) { + String cl = classes.next(); + JsonNode ids = node.get(cl); + + if (ids == null) { + continue; + } + + /* Add class label to map if it's not already there. */ + if (!this.map.containsKey(cl)) { + this.map.put(cl, new HashSet<>()); + } + + if (ids.isArray()) { + for (JsonNode item : ids) { + if (this.inverseMap.containsKey(cl)) { + throw new EvaluationException("Could not create ground truth due to a structural error: An ID must not belong to more than one class."); + } + this.map.get(cl).add(item.textValue()); + this.inverseMap.put(item.textValue(), cl); } + } } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/Copier.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/Copier.java index e8f3f8060..59f5f3890 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/Copier.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/Copier.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.standalone.importer; +import java.util.ArrayList; +import java.util.Map; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; @@ -9,129 +12,125 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.monitoring.ImportTaskMonitor; -import java.util.ArrayList; -import java.util.Map; -import java.util.Set; - /** * Copies data from an {@link Importer} to a {@link PersistencyWriter} */ public class Copier implements AutoCloseable { - private final String entityName; - private final Importer importer; - private final PersistencyWriter writer; - private static final Logger LOGGER = LogManager.getLogger(); + private final String entityName; + private final Importer importer; + private final PersistencyWriter writer; + private static final Logger LOGGER = LogManager.getLogger(); - public Copier(String entityName, Importer importer) { - this(entityName, importer, Config.sharedConfig().getDatabase().getWriterSupplier().get()); - } + public Copier(String entityName, Importer importer) { + this(entityName, importer, Config.sharedConfig().getDatabase().getWriterSupplier().get()); + } + + public Copier(String entityName, Importer importer, PersistencyWriter writer) { + this.entityName = entityName; + this.importer = importer; + this.writer = writer; + } + + public void copy() { + this.copyFrom(this.importer); + } - public Copier(String entityName, Importer importer, PersistencyWriter writer) { - this.entityName = entityName; - this.importer = importer; - this.writer = writer; + public void copyFrom(Importer importer) { + Map map = importer.readNextAsMap(); + + if (map == null) { + return; } - public void copy() { - this.copyFrom(this.importer); + Set keyset = map.keySet(); + String[] names = new String[keyset.size()]; + + int i = 0; + for (String name : keyset) { + names[i++] = name; } - public void copyFrom(Importer importer) { - Map map = importer.readNextAsMap(); + this.writer.open(entityName); + this.writer.setFieldNames(names); - if (map == null) { - return; - } + Object[] objects = new Object[names.length]; - Set keyset = map.keySet(); - String[] names = new String[keyset.size()]; + do { + for (i = 0; i < names.length; ++i) { + objects[i] = PrimitiveTypeProvider.getObject(map.get(names[i])); + } + persistTuple(this.writer.generateTuple(objects)); + } while ((map = importer.readNextAsMap()) != null); - int i = 0; - for (String name : keyset) { - names[i++] = name; - } + } - this.writer.open(entityName); - this.writer.setFieldNames(names); + private void persistTuple(PersistentTuple tuple) { + this.writer.persist(tuple); + } - Object[] objects = new Object[names.length]; + public void copyBatched(int batchSize) { + this.copyBatchedFrom(batchSize, this.importer); + } - do { - for (i = 0; i < names.length; ++i) { - objects[i] = PrimitiveTypeProvider.getObject(map.get(names[i])); - } - persistTuple(this.writer.generateTuple(objects)); - } while ((map = importer.readNextAsMap()) != null); + public void copyBatchedFrom(int batchSize, Importer importer) { + if (batchSize <= 0) { + copy(); + return; } - private void persistTuple(PersistentTuple tuple) { - this.writer.persist(tuple); + Map map = importer.readNextAsMap(); + + if (map == null) { + return; } - public void copyBatched(int batchSize) { - this.copyBatchedFrom(batchSize, this.importer); + Set keyset = map.keySet(); + String[] names = new String[keyset.size()]; + + int i = 0; + for (String name : keyset) { + names[i++] = name; } - public void copyBatchedFrom(int batchSize, Importer importer) { - - if (batchSize <= 0) { - copy(); - return; - } - - Map map = importer.readNextAsMap(); - - if (map == null) { - return; - } - - Set keyset = map.keySet(); - String[] names = new String[keyset.size()]; - - int i = 0; - for (String name : keyset) { - names[i++] = name; - } - - this.writer.open(entityName); - this.writer.setFieldNames(names); - - Object[] objects = new Object[names.length]; - - ArrayList tupleCache = new ArrayList<>(batchSize); - - long start = System.currentTimeMillis(); - ImportTaskMonitor.reportImportProgress(0, entityName, 0); - do { - for (i = 0; i < names.length; ++i) { - objects[i] = PrimitiveTypeProvider.getObject(map.get(names[i])); - } - PersistentTuple tuple = this.writer.generateTuple(objects); - tupleCache.add(tuple); - if (tupleCache.size() >= batchSize) { - int size = tupleCache.size(); - this.writer.persist(tupleCache); - long stop = System.currentTimeMillis(); - LOGGER.trace("Inserted {} elements in {} ms", size, stop-start); - ImportTaskMonitor.reportImportProgress(size, entityName, stop - start); - tupleCache.clear(); - start = System.currentTimeMillis(); - } - - } while ((map = importer.readNextAsMap()) != null); + this.writer.open(entityName); + this.writer.setFieldNames(names); + + Object[] objects = new Object[names.length]; + + ArrayList tupleCache = new ArrayList<>(batchSize); + + long start = System.currentTimeMillis(); + ImportTaskMonitor.reportImportProgress(0, entityName, 0); + do { + for (i = 0; i < names.length; ++i) { + objects[i] = PrimitiveTypeProvider.getObject(map.get(names[i])); + } + PersistentTuple tuple = this.writer.generateTuple(objects); + tupleCache.add(tuple); + if (tupleCache.size() >= batchSize) { int size = tupleCache.size(); this.writer.persist(tupleCache); long stop = System.currentTimeMillis(); - LOGGER.trace("Inserted {} elements in {} ms", size, stop-start); + LOGGER.trace("Inserted {} elements in {} ms", size, stop - start); ImportTaskMonitor.reportImportProgress(size, entityName, stop - start); + tupleCache.clear(); + start = System.currentTimeMillis(); + } - } + } while ((map = importer.readNextAsMap()) != null); + int size = tupleCache.size(); + this.writer.persist(tupleCache); + long stop = System.currentTimeMillis(); + LOGGER.trace("Inserted {} elements in {} ms", size, stop - start); + ImportTaskMonitor.reportImportProgress(size, entityName, stop - start); + } - @Override - public void close() { - this.writer.close(); - } + + @Override + public void close() { + this.writer.close(); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/LIREImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/LIREImporter.java index 091fb9cd6..2beb9c61b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/LIREImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/LIREImporter.java @@ -1,18 +1,17 @@ package org.vitrivr.cineast.standalone.importer; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; public class LIREImporter implements Importer { @@ -85,6 +84,7 @@ public Map convert(StringFloatArrayPair data) { } class StringFloatArrayPair { + final String id; final float[] vector; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java index b412e882c..e0a1e791f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/PlainTextImporter.java @@ -1,61 +1,63 @@ package org.vitrivr.cineast.standalone.importer; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; -public class PlainTextImporter implements Importer> { - /** Iterator over the files that should be imported. */ - private final Iterator files; +public class PlainTextImporter implements Importer> { - /** - * Constructor for {@link PlainTextImporter}. - * - * @param input Path to the input file or folder. - */ - public PlainTextImporter(Path input) throws IOException { - this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); - } + /** + * Iterator over the files that should be imported. + */ + private final Iterator files; - /** - * Returns the ID and the content of the next text file. - * - * @return Pair mapping an ID (from the filename) to the content of the file (text). - */ - @Override - public Pair readNext() { - try { - if (this.files.hasNext()) { - final Path path = this.files.next(); - final String segmentId = path.getFileName().toString().replace(".txt", ""); - final String text = new String(Files.readAllBytes(path)); - return new Pair<>(segmentId, text); - } else { - return null; - } - } catch (IOException e) { - return null; - } - } + /** + * Constructor for {@link PlainTextImporter}. + * + * @param input Path to the input file or folder. + */ + public PlainTextImporter(Path input) throws IOException { + this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); + } - /** - * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class - * - * @param data Pair mapping an ID to a text. - * @return HashMap containing the data. - */ - @Override - public Map convert(Pair data) { - final HashMap map = new HashMap<>(2); - map.put("id", PrimitiveTypeProvider.fromObject(data.first)); - map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); - return map; + /** + * Returns the ID and the content of the next text file. + * + * @return Pair mapping an ID (from the filename) to the content of the file (text). + */ + @Override + public Pair readNext() { + try { + if (this.files.hasNext()) { + final Path path = this.files.next(); + final String segmentId = path.getFileName().toString().replace(".txt", ""); + final String text = new String(Files.readAllBytes(path)); + return new Pair<>(segmentId, text); + } else { + return null; + } + } catch (IOException e) { + return null; } + } + + /** + * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class + * + * @param data Pair mapping an ID to a text. + * @return HashMap containing the data. + */ + @Override + public Map convert(Pair data) { + final HashMap map = new HashMap<>(2); + map.put("id", PrimitiveTypeProvider.fromObject(data.first)); + map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); + return map; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java index adbc89a29..a1d9da549 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/TagsFulltextImporter.java @@ -1,64 +1,66 @@ package org.vitrivr.cineast.standalone.importer; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; /** * Imports the tag files files generated for the collection for the VBS challenge to an entity for fulltext retrieval. */ -public class TagsFulltextImporter implements Importer> { - /** Iterator over the files that should be imported. */ - private final Iterator files; +public class TagsFulltextImporter implements Importer> { - /** - * Constructor for {@link PlainTextImporter}. - * - * @param input Path to the input file or folder. - */ - public TagsFulltextImporter(Path input) throws IOException { - this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); - } + /** + * Iterator over the files that should be imported. + */ + private final Iterator files; - /** - * Returns the ID and the content of the next text file. - * - * @return Pair mapping an ID (from the filename) to the content of the file (text). - */ - @Override - public Pair readNext() { - try { - if (this.files.hasNext()) { - final Path path = this.files.next(); - final String segmentId = path.getFileName().toString().replace(".txt", ""); - final String text = new String(Files.readAllBytes(path)).replaceAll("\\s+", " ").trim(); - return new Pair<>(segmentId, text); - } else { - return null; - } - } catch (IOException e) { - return null; - } - } + /** + * Constructor for {@link PlainTextImporter}. + * + * @param input Path to the input file or folder. + */ + public TagsFulltextImporter(Path input) throws IOException { + this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".txt")).iterator(); + } - /** - * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class - * - * @param data Pair mapping an ID to a text. - * @return HashMap containing the data. - */ - @Override - public Map convert(Pair data) { - final HashMap map = new HashMap<>(2); - map.put("id", PrimitiveTypeProvider.fromObject(data.first)); - map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); - return map; + /** + * Returns the ID and the content of the next text file. + * + * @return Pair mapping an ID (from the filename) to the content of the file (text). + */ + @Override + public Pair readNext() { + try { + if (this.files.hasNext()) { + final Path path = this.files.next(); + final String segmentId = path.getFileName().toString().replace(".txt", ""); + final String text = new String(Files.readAllBytes(path)).replaceAll("\\s+", " ").trim(); + return new Pair<>(segmentId, text); + } else { + return null; + } + } catch (IOException e) { + return null; } + } + + /** + * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class + * + * @param data Pair mapping an ID to a text. + * @return HashMap containing the data. + */ + @Override + public Map convert(Pair data) { + final HashMap map = new HashMap<>(2); + map.put("id", PrimitiveTypeProvider.fromObject(data.first)); + map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); + return map; + } } \ No newline at end of file diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java index ee2afb329..21ed8c545 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/VbsMetaImporter.java @@ -1,68 +1,69 @@ package org.vitrivr.cineast.standalone.importer; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; /** - * Imports the XML metadata files bundled with the collection for the VBS challenge to an entity for fulltext retrieval. - * The XML structure in the files is completely ommited and only text is imported. + * Imports the XML metadata files bundled with the collection for the VBS challenge to an entity for fulltext retrieval. The XML structure in the files is completely ommited and only text is imported. */ -public class VbsMetaImporter implements Importer> { - /** Iterator over the files that should be imported. */ - private final Iterator files; +public class VbsMetaImporter implements Importer> { - /** - * Constructor for {@link PlainTextImporter}. - * - * @param input Path to the input file or folder. - */ - public VbsMetaImporter(Path input) throws IOException { - this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".xml")).iterator(); - } + /** + * Iterator over the files that should be imported. + */ + private final Iterator files; - /** - * Returns the ID and the content of the next XML file. - * - * @return Pair mapping an ID (from the filename) to the content of the file (text). - */ - @Override - public Pair readNext() { - try { - if (this.files.hasNext()) { - final Path path = this.files.next(); - final String objectId = path.getFileName().toString().replace(".xml", ""); - final String text = new String(Files.readAllBytes(path)) - .replaceAll("<[^>]+>", "\n") - .replaceAll("\\s+"," ") - .trim(); - return new Pair<>(objectId, text); - } else { - return null; - } - } catch (IOException e) { - return null; - } - } + /** + * Constructor for {@link PlainTextImporter}. + * + * @param input Path to the input file or folder. + */ + public VbsMetaImporter(Path input) throws IOException { + this.files = Files.walk(input, 2).filter(s -> s.toString().endsWith(".xml")).iterator(); + } - /** - * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class - * - * @param data Pair mapping an ID to a text. - * @return HashMap containing the data. - */ - @Override - public Map convert(Pair data) { - final HashMap map = new HashMap<>(2); - map.put("id", PrimitiveTypeProvider.fromObject(data.first)); - map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); - return map; + /** + * Returns the ID and the content of the next XML file. + * + * @return Pair mapping an ID (from the filename) to the content of the file (text). + */ + @Override + public Pair readNext() { + try { + if (this.files.hasNext()) { + final Path path = this.files.next(); + final String objectId = path.getFileName().toString().replace(".xml", ""); + final String text = new String(Files.readAllBytes(path)) + .replaceAll("<[^>]+>", "\n") + .replaceAll("\\s+", " ") + .trim(); + return new Pair<>(objectId, text); + } else { + return null; + } + } catch (IOException e) { + return null; } + } + + /** + * Converts a mapping from ID to text content to a HashMap representation that can be used by the {@link Copier} class + * + * @param data Pair mapping an ID to a text. + * @return HashMap containing the data. + */ + @Override + public Map convert(Pair data) { + final HashMap map = new HashMap<>(2); + map.put("id", PrimitiveTypeProvider.fromObject(data.first)); + map.put("feature", PrimitiveTypeProvider.fromObject(data.second)); + return map; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java index 93995859b..bb5291c85 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/AsrDataImportHandler.java @@ -1,26 +1,26 @@ package org.vitrivr.cineast.standalone.importer.handlers; +import java.io.IOException; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.importer.PlainTextImporter; -import java.io.IOException; -import java.nio.file.Path; - public class AsrDataImportHandler extends DataImportHandler { - private static final Logger LOGGER = LogManager.getLogger(); - public AsrDataImportHandler(int threads, int batchsize) { - super(threads, batchsize); - } + private static final Logger LOGGER = LogManager.getLogger(); + + public AsrDataImportHandler(int threads, int batchsize) { + super(threads, batchsize); + } - @Override - public void doImport(Path path) { - try { - this.futures.add(this.service.submit(new DataImportRunner(new PlainTextImporter(path), "features_asr", "asr"))); - } catch (IOException e) { - LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); - } + @Override + public void doImport(Path path) { + try { + this.futures.add(this.service.submit(new DataImportRunner(new PlainTextImporter(path), "features_asr", "asr"))); + } catch (IOException e) { + LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java index 2556347c6..df3a1f6ca 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java @@ -7,7 +7,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; @@ -21,213 +20,209 @@ import org.vitrivr.cineast.standalone.importer.Copier; import org.vitrivr.cineast.standalone.monitoring.ImportTaskMonitor; import org.vitrivr.cottontail.client.language.ddl.CreateEntity; -import org.vitrivr.cottontail.grpc.CottontailGrpc; public abstract class DataImportHandler { - /** - * Drops the entity if called. This is in order to have clean imports. - * @param entityName The entity name to drop - * @param taskName The task name during which this dropping occurs. Only for logging purposes - */ - protected static void cleanOnDemand(String entityName, String taskName){ - final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); - /* Beware, this drops the table */ - CreateEntity createEntity = null; - CottontailWrapper cottontail = null; - if (Config.sharedConfig().getDatabase().getSelector() != DatabaseConfig.Selector.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != DatabaseConfig.Writer.COTTONTAIL) { - LOGGER.warn("Other database than Cottontail DB in use. Using inconvenient database restore"); - }else{ - LOGGER.info("Storing entity ({}) details for re-setup", entityName); - cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); - //entityDefinition = cottontail.entityDetailsBlocking(CottontailMessageBuilder.entity(entityName)); - } - LOGGER.info("{} - Dropping table for entity {}...", taskName, entityName); - ec.dropEntity(entityName); - LOGGER.info("{} - Finished dropping table for entity {}", taskName, entityName); - if (createEntity == null) { - LOGGER.warn("Calling command: setup -- This may take a while"); - DatabaseSetupCommand setupCmd = new DatabaseSetupCommand(); - setupCmd.doSetup(); - } else { - cottontail.client.create(createEntity, null); - LOGGER.info("Re-created entity: {}", createEntity.getBuilder().getDefinition().getEntity().getName()); - } + /** + * Drops the entity if called. This is in order to have clean imports. + * + * @param entityName The entity name to drop + * @param taskName The task name during which this dropping occurs. Only for logging purposes + */ + protected static void cleanOnDemand(String entityName, String taskName) { + final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); + /* Beware, this drops the table */ + CreateEntity createEntity = null; + CottontailWrapper cottontail = null; + if (Config.sharedConfig().getDatabase().getSelector() != DatabaseConfig.Selector.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != DatabaseConfig.Writer.COTTONTAIL) { + LOGGER.warn("Other database than Cottontail DB in use. Using inconvenient database restore"); + } else { + LOGGER.info("Storing entity ({}) details for re-setup", entityName); + cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); + //entityDefinition = cottontail.entityDetailsBlocking(CottontailMessageBuilder.entity(entityName)); } - - protected static void createEntityOnDemand(EntityDefinition def, String taskName){ - LOGGER.debug("Creating entity: "+def); - EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); - if(ec.existsEntity(def.getEntityName())){ - LOGGER.warn("Entity already exists, ignoring"); - return; - } - if(ec.createEntity(def)){ - LOGGER.info("Successfully created entity "+def); - }else{ - LOGGER.error("Could not create entity "+def+" please see the log"); - } + LOGGER.info("{} - Dropping table for entity {}...", taskName, entityName); + ec.dropEntity(entityName); + LOGGER.info("{} - Finished dropping table for entity {}", taskName, entityName); + if (createEntity == null) { + LOGGER.warn("Calling command: setup -- This may take a while"); + DatabaseSetupCommand setupCmd = new DatabaseSetupCommand(); + setupCmd.doSetup(); + } else { + cottontail.client.create(createEntity, null); + LOGGER.info("Re-created entity: {}", createEntity.getBuilder().getDefinition().getEntity().getName()); } + } + + protected static void createEntityOnDemand(EntityDefinition def, String taskName) { + LOGGER.debug("Creating entity: " + def); + EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); + if (ec.existsEntity(def.getEntityName())) { + LOGGER.warn("Entity already exists, ignoring"); + return; + } + if (ec.createEntity(def)) { + LOGGER.info("Successfully created entity " + def); + } else { + LOGGER.error("Could not create entity " + def + " please see the log"); + } + } + /** + * This inner class implements the runnable that actually executes the copy operation. + */ + protected class DataImportRunner implements Runnable { /** - * This inner class implements the runnable that actually executes the copy operation. + * Name of the entity the {@link DataImportRunner} populates. */ - protected class DataImportRunner implements Runnable { - - /** - * Name of the entity the {@link DataImportRunner} populates. - */ - private final String entityName; - - /** - * The {@link Importer} instance used to import the data. - */ - private final Importer importer; - /** - * A -possibly- human readable name for the import task - */ - private final String taskName; - - /** - * Whether or not the table of the entity to import should be dropped beforehand. - * Basically, if this is true: its a TRUNCATE_EXISTING write, otherwise it's an APPEND write - */ - private final boolean clean; - - /** - * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}. - * If specified, drops the entity's table beforehand. - * - * @param importer The {@link Importer} to run the import of - * @param entityName The name of the entity to import into - * @param taskName The name of the task (possibly human readable) - * @param clean Whether to drop the entity's table beforehand or not - */ - public DataImportRunner(Importer importer, String entityName, String taskName, boolean clean) { - this.entityName = entityName; - this.importer = importer; - this.taskName = taskName; - this.clean = clean; - if(clean){ - cleanOnDemand(this.entityName, this.taskName); - } - } - - /** - * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}. - * Does an APPEND import, i.e. existing entries on that entity are kept. - * - * @param importer The {@link Importer} to run the import of - * @param entityName The name of the entity to import into - * @param taskName The name of the task (possibly human readable) - */ - public DataImportRunner(Importer importer, String entityName, String taskName) { - this(importer, entityName, taskName, false); - } - - /** - * Creates a new import runner for given importer and creates the entity, if not existent. - * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}, while creating the entity before hand, using the given {@link EntityDefinition}. - * @param importer The importer to run the import of - * @param entity The entity definition of the entity to import. If not existent, the entity will be created first - * @param taskName The name of the task, possibly human readable - */ - public DataImportRunner(Importer importer, EntityDefinition entity, String taskName){ - this(importer, entity.getEntityName(), taskName); - createEntityOnDemand(entity, taskName); - } - - /** - * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's run method to be called in that separately executing thread. - *

    - * The general contract of the method run is that it may take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - try { - long start = System.currentTimeMillis(); - final Copier copier = new Copier(this.entityName, this.importer); - LOGGER.info("Starting import on entity: {} with importer {}, task {}...", this.entityName, this.importer.getClass().getSimpleName(), taskName); - copier.copyBatched(DataImportHandler.this.batchsize); - copier.close(); - LOGGER.info("Completed import of entity: {}, task {}", this.entityName, taskName); - long stop = System.currentTimeMillis(); - ImportTaskMonitor.reportExecutionTime(taskName, stop - start); - Thread.sleep(1_000); - } catch (Exception e) { - LOGGER.error("Error for task {} while copying data for '{}': {}", taskName, this.entityName, LogHelper.getStackTrace(e)); - } - } - } - - private static final Logger LOGGER = LogManager.getLogger(); + private final String entityName; /** - * ExecutorService used for execution of the {@link DataImportRunner}. + * The {@link Importer} instance used to import the data. */ - protected final ExecutorService service; - + private final Importer importer; /** - * Size of data batches (i.e. number if tuples) that are sent to the persistence layer. + * A -possibly- human readable name for the import task */ - protected final int batchsize; + private final String taskName; - protected int numberOfThreads; + /** + * Whether or not the table of the entity to import should be dropped beforehand. Basically, if this is true: its a TRUNCATE_EXISTING write, otherwise it's an APPEND write + */ + private final boolean clean; /** + * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}. If specified, drops the entity's table beforehand. * + * @param importer The {@link Importer} to run the import of + * @param entityName The name of the entity to import into + * @param taskName The name of the task (possibly human readable) + * @param clean Whether to drop the entity's table beforehand or not */ - protected final ArrayList> futures = new ArrayList<>(); + public DataImportRunner(Importer importer, String entityName, String taskName, boolean clean) { + this.entityName = entityName; + this.importer = importer; + this.taskName = taskName; + this.clean = clean; + if (clean) { + cleanOnDemand(this.entityName, this.taskName); + } + } /** - * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. + * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}. Does an APPEND import, i.e. existing entries on that entity are kept. * - * @param threads Number of threads to use for data import. - * @param batchsize Size of data batches that are sent to the persistence layer. + * @param importer The {@link Importer} to run the import of + * @param entityName The name of the entity to import into + * @param taskName The name of the task (possibly human readable) */ - public DataImportHandler(int threads, int batchsize) { - this.service = Executors.newFixedThreadPool(threads); - this.batchsize = batchsize; - this.numberOfThreads = threads; + public DataImportRunner(Importer importer, String entityName, String taskName) { + this(importer, entityName, taskName, false); } - public abstract void doImport(Path path); + /** + * Creates a new import runner for given importer and creates the entity, if not existent. Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}, while creating the entity before hand, using the given {@link EntityDefinition}. + * + * @param importer The importer to run the import of + * @param entity The entity definition of the entity to import. If not existent, the entity will be created first + * @param taskName The name of the task, possibly human readable + */ + public DataImportRunner(Importer importer, EntityDefinition entity, String taskName) { + this(importer, entity.getEntityName(), taskName); + createEntityOnDemand(entity, taskName); + } /** - * Awaits completion of the individual Futures. This method blocks until all Futures have been completed. + * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's run method to be called in that separately executing thread. + *

    + * The general contract of the method run is that it may take any action whatsoever. + * + * @see Thread#run() */ - public void waitForCompletion() { - this.futures.removeIf(f -> { - try { - Object o = f.get(); - if (o == null) { - return true; - } - LOGGER.warn("Future returned {}, still returning true", o); - return true; - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); - LOGGER.error("Execution of one of the tasks could not be completed!"); - return true; - } - }); - try { - this.futures.forEach(future -> { - LOGGER.warn("A future is still present, this should not be happening."); - }); - LOGGER.info("Shutting down threadpool for {}", this.getClass().getSimpleName()); - this.service.shutdown(); - LOGGER.info("Awaiting termination {}", this.getClass().getSimpleName()); - this.service.awaitTermination(30, TimeUnit.SECONDS); - LOGGER.info("Service terminated {}", this.getClass().getSimpleName()); - } catch (InterruptedException e) { - e.printStackTrace(); + @Override + public void run() { + try { + long start = System.currentTimeMillis(); + final Copier copier = new Copier(this.entityName, this.importer); + LOGGER.info("Starting import on entity: {} with importer {}, task {}...", this.entityName, this.importer.getClass().getSimpleName(), taskName); + copier.copyBatched(DataImportHandler.this.batchsize); + copier.close(); + LOGGER.info("Completed import of entity: {}, task {}", this.entityName, taskName); + long stop = System.currentTimeMillis(); + ImportTaskMonitor.reportExecutionTime(taskName, stop - start); + Thread.sleep(1_000); + } catch (Exception e) { + LOGGER.error("Error for task {} while copying data for '{}': {}", taskName, this.entityName, LogHelper.getStackTrace(e)); + } + } + } + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * ExecutorService used for execution of the {@link DataImportRunner}. + */ + protected final ExecutorService service; + + /** + * Size of data batches (i.e. number if tuples) that are sent to the persistence layer. + */ + protected final int batchsize; + + protected int numberOfThreads; + + /** + * + */ + protected final ArrayList> futures = new ArrayList<>(); + + /** + * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. + * + * @param threads Number of threads to use for data import. + * @param batchsize Size of data batches that are sent to the persistence layer. + */ + public DataImportHandler(int threads, int batchsize) { + this.service = Executors.newFixedThreadPool(threads); + this.batchsize = batchsize; + this.numberOfThreads = threads; + } + + public abstract void doImport(Path path); + + /** + * Awaits completion of the individual Futures. This method blocks until all Futures have been completed. + */ + public void waitForCompletion() { + this.futures.removeIf(f -> { + try { + Object o = f.get(); + if (o == null) { + return true; } + LOGGER.warn("Future returned {}, still returning true", o); + return true; + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + LOGGER.error("Execution of one of the tasks could not be completed!"); + return true; + } + }); + try { + this.futures.forEach(future -> { + LOGGER.warn("A future is still present, this should not be happening."); + }); + LOGGER.info("Shutting down threadpool for {}", this.getClass().getSimpleName()); + this.service.shutdown(); + LOGGER.info("Awaiting termination {}", this.getClass().getSimpleName()); + this.service.awaitTermination(30, TimeUnit.SECONDS); + LOGGER.info("Service terminated {}", this.getClass().getSimpleName()); + } catch (InterruptedException e) { + e.printStackTrace(); } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java index 5111be2e5..db58a7ea2 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java @@ -1,44 +1,46 @@ package org.vitrivr.cineast.standalone.importer.handlers; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.util.LogHelper; -import org.vitrivr.cineast.core.importer.JsonObjectImporter; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.importer.JsonObjectImporter; +import org.vitrivr.cineast.core.util.LogHelper; public class JsonDataImportHandler extends DataImportHandler { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); - public JsonDataImportHandler(int threads, int batchsize) { - super(threads, batchsize); - } + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); + + public JsonDataImportHandler(int threads, int batchsize) { + super(threads, batchsize); + } - /** - * Starts data import process using JSON files. The method can either be used with a single file or a folder containing JSON files. - * - * @param path Path to the JSON file or a folder containing JSON files. - */ - @Override - public void doImport(Path path) { + /** + * Starts data import process using JSON files. The method can either be used with a single file or a folder containing JSON files. + * + * @param path Path to the JSON file or a folder containing JSON files. + */ + @Override + public void doImport(Path path) { + try { + LOGGER.info("Starting data import with JSON files in: {}", path.toString()); + Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> { + final String filename = p.getFileName().toString(); + final String suffix = filename.substring(filename.lastIndexOf(".")); try { - LOGGER.info("Starting data import with JSON files in: {}", path.toString()); - Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> { - final String filename = p.getFileName().toString(); - final String suffix = filename.substring(filename.lastIndexOf(".")); - try { - this.futures.add(this.service.submit(new DataImportRunner(new JsonObjectImporter(p.toFile()), filename.replace(suffix, ""), "json_" + filename.replace(suffix, "")))); - } catch (IOException e) { - LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString()); - } - }); - this.waitForCompletion(); - LOGGER.info("Completed data import with JSON files in: {}", path.toString()); + this.futures.add(this.service.submit(new DataImportRunner(new JsonObjectImporter(p.toFile()), filename.replace(suffix, ""), "json_" + filename.replace(suffix, "")))); } catch (IOException e) { - LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); + LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString()); } + }); + this.waitForCompletion(); + LOGGER.info("Completed data import with JSON files in: {}", path.toString()); + } catch (IOException e) { + LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/LIREImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/LIREImportHandler.java index b425fbb04..2a5e7f073 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/LIREImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/LIREImportHandler.java @@ -9,40 +9,43 @@ import org.vitrivr.cineast.standalone.importer.LIREImporter; public class LIREImportHandler extends DataImportHandler { - /** */ - private static final Logger LOGGER = LogManager.getLogger(); - /** - * Default constructor for {@link LIREImportHandler}. - * - * @param threads Number if threads to use - * @param batchSize Batch size to use for import. - */ - public LIREImportHandler(int threads, int batchSize) { - super(threads, batchSize); - } + /** + * + */ + private static final Logger LOGGER = LogManager.getLogger(); - @Override - public void doImport(Path path) { - try { - LOGGER.info("Starting data import with LIRE files in: {}", path.toString()); - if (!Files.isDirectory(path)) { - LOGGER.error("Could not start data import process with path '{}', because it is not a directory.", path); - return; - } + /** + * Default constructor for {@link LIREImportHandler}. + * + * @param threads Number if threads to use + * @param batchSize Batch size to use for import. + */ + public LIREImportHandler(int threads, int batchSize) { + super(threads, batchSize); + } - final String name = path.getFileName().toString(); - Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith("." + name)).forEach(p -> { - final String filename = p.getFileName().toString(); - final String suffix = filename.substring(filename.lastIndexOf(".")); - try { - this.futures.add(this.service.submit(new DataImportRunner(new LIREImporter(p.toFile()), "feature_" + name, "lire_" + filename.replace(suffix, "")))); - } catch (IOException e) { - LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString()); - } - }); + @Override + public void doImport(Path path) { + try { + LOGGER.info("Starting data import with LIRE files in: {}", path.toString()); + if (!Files.isDirectory(path)) { + LOGGER.error("Could not start data import process with path '{}', because it is not a directory.", path); + return; + } + + final String name = path.getFileName().toString(); + Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith("." + name)).forEach(p -> { + final String filename = p.getFileName().toString(); + final String suffix = filename.substring(filename.lastIndexOf(".")); + try { + this.futures.add(this.service.submit(new DataImportRunner(new LIREImporter(p.toFile()), "feature_" + name, "lire_" + filename.replace(suffix, "")))); } catch (IOException e) { - LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); + LOGGER.error("Could not start data import for file '{}'. Skipping...?", p.toString()); } + }); + } catch (IOException e) { + LOGGER.error("Could not start data import process with path '{}' due to an IOException: {}. Aborting...", path.toString(), LogHelper.getStackTrace(e)); } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java index 82fe92399..1dbf72573 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/OcrDataImportHandler.java @@ -1,17 +1,18 @@ package org.vitrivr.cineast.standalone.importer.handlers; +import java.io.IOException; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.features.OCRSearch; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.importer.PlainTextImporter; -import java.io.IOException; -import java.nio.file.Path; - public class OcrDataImportHandler extends DataImportHandler { - /** */ + /** + * + */ private static final Logger LOGGER = LogManager.getLogger(); public OcrDataImportHandler(int threads, int batchsize) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java index 1ccd99c97..6f53f4e4c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/ProtoDataImportHandler.java @@ -1,13 +1,12 @@ package org.vitrivr.cineast.standalone.importer.handlers; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.importer.TupleInsertMessageImporter; - import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.importer.TupleInsertMessageImporter; public class ProtoDataImportHandler extends DataImportHandler { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImportHandler.java index 3d5be2ded..6bf6ca965 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImportHandler.java @@ -1,29 +1,28 @@ package org.vitrivr.cineast.standalone.importer.lsc2020; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.features.DescriptionTextSearch; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.nio.file.Path; - public class CaptionImportHandler extends DataImportHandler { - private static final Logger LOGGER = LogManager.getLogger(CaptionImportHandler.class); + private static final Logger LOGGER = LogManager.getLogger(CaptionImportHandler.class); - /** - * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. - * - * @param threads Number of threads to use for data import. - * @param batchsize Size of data batches that are sent to the persistence layer. - */ - public CaptionImportHandler(int threads, int batchsize) { - super(threads, batchsize); - } + /** + * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. + * + * @param threads Number of threads to use for data import. + * @param batchsize Size of data batches that are sent to the persistence layer. + */ + public CaptionImportHandler(int threads, int batchsize) { + super(threads, batchsize); + } - @Override - public void doImport(Path path) { - LOGGER.info("Starting caption import"); - this.futures.add(this.service.submit(new DataImportRunner(new CaptionImporter(path), DescriptionTextSearch.DESCRIPTION_TEXT_TABLE_NAME, "lsc-caption"))); - } + @Override + public void doImport(Path path) { + LOGGER.info("Starting caption import"); + this.futures.add(this.service.submit(new DataImportRunner(new CaptionImporter(path), DescriptionTextSearch.DESCRIPTION_TEXT_TABLE_NAME, "lsc-caption"))); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImporter.java index c94c04610..4d82d1cc4 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/CaptionImporter.java @@ -2,13 +2,6 @@ import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -17,57 +10,63 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; public class CaptionImporter implements Importer> { - private static final Logger LOGGER = LogManager.getLogger(CaptionImporter.class); + private static final Logger LOGGER = LogManager.getLogger(CaptionImporter.class); - public static final String CAPTIONS_FILE_NAME = "lsc2020-captions.csv"; + public static final String CAPTIONS_FILE_NAME = "lsc2020-captions.csv"; - public static final int ID_COLUMN = 1; - public static final int CAPTION_COLUMN = 2; + public static final int ID_COLUMN = 1; + public static final int CAPTION_COLUMN = 2; - private Path root; + private Path root; - public CaptionImporter(Path root) { - this.root = root; - try { - prepareFile(); - } catch (IOException | CsvException e) { - throw LOGGER.throwing(Level.ERROR, new RuntimeException("Could not preprae", e)); - } + public CaptionImporter(Path root) { + this.root = root; + try { + prepareFile(); + } catch (IOException | CsvException e) { + throw LOGGER.throwing(Level.ERROR, new RuntimeException("Could not preprae", e)); } + } - private Iterator iterator; + private Iterator iterator; - private void prepareFile() throws IOException, CsvException { - long start = System.currentTimeMillis(); - Path file = root.resolve(CAPTIONS_FILE_NAME); - CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); - List metadataFileContents = csv.readAll(); - metadataFileContents.remove(0); // Headers not required - iterator = metadataFileContents.iterator(); - LOGGER.info("Preparation done in {}ms", (System.currentTimeMillis() - start)); - } + private void prepareFile() throws IOException, CsvException { + long start = System.currentTimeMillis(); + Path file = root.resolve(CAPTIONS_FILE_NAME); + CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); + List metadataFileContents = csv.readAll(); + metadataFileContents.remove(0); // Headers not required + iterator = metadataFileContents.iterator(); + LOGGER.info("Preparation done in {}ms", (System.currentTimeMillis() - start)); + } - private Map parseLine(String[] line) { - final HashMap map = new HashMap<>(2); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(LSCUtilities.pathToSegmentId(line[ID_COLUMN]))); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(line[CAPTION_COLUMN])); - return map; - } + private Map parseLine(String[] line) { + final HashMap map = new HashMap<>(2); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(LSCUtilities.pathToSegmentId(line[ID_COLUMN]))); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(line[CAPTION_COLUMN])); + return map; + } - @Override - public Map readNext() { - if (iterator.hasNext()) { - return parseLine(iterator.next()); - } - return null; + @Override + public Map readNext() { + if (iterator.hasNext()) { + return parseLine(iterator.next()); } + return null; + } - @Override - public Map convert(Map data) { - return data; - } + @Override + public Map convert(Map data) { + return data; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/JsonTagReader.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/JsonTagReader.java index 5bc50bf86..fb7023255 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/JsonTagReader.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/JsonTagReader.java @@ -15,24 +15,15 @@ /** * Reader for the tag format of tags, published by the MySceal Team in 2021. - * + *

    * Requires a JSON file (published in 2021 at http://lsc.dcu.ie/resources/microsoft_tags.json - * + *

    * Format: * * - * { - * "item1":{ - * "tag1": score, - * "tag2": score - * }, - * "item2":{ - * "tag1":score, - * "tag2":score - * } - * } + * { "item1":{ "tag1": score, "tag2": score }, "item2":{ "tag1":score, "tag2":score } } * - * + *

    * {@code item} is formatted as filename, which {@link LSCUtilities#pathToSegmentId(String)} translates into vitrivr id */ public class JsonTagReader { @@ -60,9 +51,9 @@ private void readEntries() throws IOException { String segmentId = LSCUtilities.pathToSegmentId(entry.getKey()); JsonNode tags = entry.getValue(); List> segmentTags = new ArrayList<>(); - if(!tags.isEmpty()){ + if (!tags.isEmpty()) { tags.fields().forEachRemaining(tagEntry -> { - if(tagEntry.getValue().isFloatingPointNumber()){ + if (tagEntry.getValue().isFloatingPointNumber()) { segmentTags.add(new Pair<>(tagEntry.getKey(), tagEntry.getValue().floatValue())); } uniqueTags.add(tagEntry.getKey()); @@ -74,11 +65,11 @@ private void readEntries() throws IOException { LOGGER.info("Successfully processed all tags"); } - public HashSet getUniqueTags(){ + public HashSet getUniqueTags() { return uniqueTags; } - public HashMap>> getTagScoreMap(){ + public HashMap>> getTagScoreMap() { return tagEntries; } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java index 5ee575409..4af312dab 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java @@ -3,31 +3,39 @@ import com.google.common.collect.Lists; import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.SetUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.time.*; -import java.util.*; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class LSCUtilities { - public static final String LSC_REPORT_VALID_FILNE_NAME = "report-valid.txt"; - public static final String METADATA_FILE_NAME = "lsc2021-metadata.csv"; - public static final String CONCEPTS_FILE_NAME = "lsc2021-visual-concepts.csv"; - public static final String META_NO_PATH_FILE = "meta-no-path.txt"; - public static final String WRITTEN_FILE = "written.txt"; - public static final String KEY_MINUTEID = "minute_id"; - public static final int CONCEPTS_MINUTEID_COL = 0; - public static final int CONCEPTS_IMAGEPATH_COL = 2; - //minute_id,utc_time,local_time,timezone,lat,lon,semantic_name,elevation,speed,heart,calories,activity_type,steps + public static final String LSC_REPORT_VALID_FILNE_NAME = "report-valid.txt"; + public static final String METADATA_FILE_NAME = "lsc2021-metadata.csv"; + public static final String CONCEPTS_FILE_NAME = "lsc2021-visual-concepts.csv"; + public static final String META_NO_PATH_FILE = "meta-no-path.txt"; + public static final String WRITTEN_FILE = "written.txt"; + public static final String KEY_MINUTEID = "minute_id"; + public static final int CONCEPTS_MINUTEID_COL = 0; + public static final int CONCEPTS_IMAGEPATH_COL = 2; + //minute_id,utc_time,local_time,timezone,lat,lon,semantic_name,elevation,speed,heart,calories,activity_type,steps /* minute_id, utc_time, @@ -43,415 +51,407 @@ public class LSCUtilities { activity_type, steps */ - public static final int META_MIN_COL = 0; - public static final String META_MIN_NAME = "minute_id"; - public static final int META_UTC_COL = 1; - public static final String META_UTC_NAME = "utc_time"; - public static final int META_LOCAL_COL = 2; - public static final String META_LOCAL_NAME = "local_time"; - public static final int META_TIMEZONE_COL = 3; - public static final String META_TIMEZONE_NAME = "timezone"; - public static final int META_LAT_COL = 4; - public static final String META_LAT_NAME = "lat"; - public static final int META_LON_COL = 5; - public static final String META_LON_NAME = "lon"; - public static final int META_SEMANTIC_COL = 6; - public static final String META_SEMANTIC_NAME = "semantic_name"; - public static final int META_ELEVATION_COL = 7; - public static final String META_ELEVATION_NAME = "elevation"; - public static final int META_SPEED_COL = 8; - public static final String META_SPEED_NAME = "speed"; - public static final int META_HEART_COL = 9; - public static final String META_HEART_NAME = "heart"; - public static final int META_CALORIES_COL = 10; - public static final String META_CALORIES_NAME = "calories"; - public static final int META_ACTIVITY_COL = 11; - public static final String META_ACTIVITY_NAME = "activity_type"; - public static final int META_STEPS_COL = 12; - public static final String META_STEPS_NAME = "steps"; - //minute_id,utc_time,local_time,timezone,lat,lon,semantic_name,elevation,speed,heart,calories,activity_type,steps - public static final String[] META_NAMES = new String[]{META_MIN_NAME, META_UTC_NAME, META_LOCAL_NAME, META_TIMEZONE_NAME, META_LAT_NAME, META_LON_NAME, META_SEMANTIC_NAME, META_ELEVATION_NAME, META_SPEED_NAME, META_HEART_NAME, META_CALORIES_NAME, META_ACTIVITY_NAME, META_STEPS_NAME - }; - public static final Set META_COLUMNS_IN_USE = new HashSet<>(Lists.newArrayList(META_MIN_COL, META_UTC_COL, META_LOCAL_COL, META_LAT_COL, META_LON_COL, META_TIMEZONE_COL, META_SEMANTIC_COL, META_ACTIVITY_COL)); - public static final String DEFAULT_DOMAIN = "LSCMETA"; - public static final String LOCATION_DOMAIN = "LOCATION"; - public static final String DATETIME_DOMAIN = "TIME"; - public static final String LSC_UTC_PREFIX = "UTC_"; - public static final String LSC_FORMAT = "yyyy-MM-dd_hh:mm"; - - public static final String PROCESSED_META_UTC = "p_utc_standard"; - public static final String PROCESSED_META_LOCAL = "p_local_standard"; - public static final String PROCESSED_META_DATETIME = "p_datetime"; // based on filename, utc - public static final String PROCESSED_META_DAY_OF_WEEK = "p_day_of_week"; - public static final String PROCESSED_META_PHASE_OF_DAY = "p_phase_of_day"; - public static final String PROCESSED_META_HOUR_OF_DAY = "p_hour"; - public static final String PROCESSED_META_MONTH = "p_month"; - public static final String PROCESSED_META_YEAR = "p_year"; - public static final String PROCESSED_META_DAY = "p_day"; - - - private static final Logger LOGGER = LogManager.getLogger(LSCUtilities.class); - - - - private static LSCUtilities instance = null; - private final Path root; - private List headerlessMetaContents; - - @Deprecated private HashMap minuteIdPathMap = new HashMap<>(); - - private HashMap filenameToMinuteIdMap = new HashMap<>(); - private HashMap metaPerMinute = new HashMap<>(); - - private LSCUtilities(final Path root) throws IOException, CsvException { - this.root = root; - // this.initLookup(minuteIdPathMap, root); - LOGGER.info("Initialising lookup..."); - this.initMap(this.readListOfFiles(root)); - LOGGER.info("Initialisation done."); - // headerlessMetaContents = readFile(root); - } - - public static LSCUtilities create(final Path root) throws IOException, CsvException { - if (instance == null) { - instance = new LSCUtilities(root); - } else if(!root.equals(instance.root)) { - return instance; - }else { - LOGGER.warn("Instance already created with path {}. Returning this one", instance.root); - } - return instance; - } - - public static LSCUtilities getInstance() { - if (instance != null) { - return instance; - } else { - throw new IllegalStateException("There's no instance, use LSCUtilities.create first"); - } - } - - /** - * Writes the list as individual lines in a textfile. - * The file must already be created - * - * @param directory The directory the file resides in - * @param file The file to write into - * @param lines The lines to write - * @throws IOException If something goes wrong. Possibly due to the file not being created beforehand - */ - public static void writeLines(Path directory, String file, List lines) throws IOException { - final Path p = directory.resolve(file); - Files.write(p, lines, StandardOpenOption.TRUNCATE_EXISTING); - } - - /** - * Converts an imagepath to a corresponding segmentId. - *
    - * In the LSC2020 dataset, image paths (from file lsc2020-visual-concepts.csv) - * do contain more than the actual (image) file, hence some conversion is required - *
    - * Prepends is_, removes anything before a slash ("/"), if present, and after a dot (".") (i.e., file extension), if present - */ - public static String pathToSegmentId(String path) { - final int beginIdx = path.contains("/") ? path.lastIndexOf("/") + 1 : 0; - final int endIdx = path.contains(".") ? path.lastIndexOf(".") : path.length(); - final String prefix = path.startsWith("is_") ? "" : "is_"; - return prefix + path.substring(beginIdx, endIdx); - } - - public static String cleanImagePath(String path) { - if (path.startsWith("DATASETS/LSC2020/")) { - return path.substring("DATASETS/LSC2020/".length() + 1); - } else { - return path; - } - } - - /** - * Two potential naming formats:
    - * Simple Format: {@code YYYMMDD_HHMMSS_000.ext} and
    - * Extended Format: {@code prefix001_prefix_YYYMMDD_HHMMSS*.ext} - * Extension is jpg - *

    - * Example of Simple: {@code 20160914_192303_000.jpg}
    - * Example of Extended: {@code B00006542_21I6X0_20180505_140053E.JPG} - * - *

    - */ - public static Optional filenameToMinuteId(String filename) { - String[] parts = filename.split("_"); - switch (parts.length) { - case 3: // simple - return Optional.of(filename.substring(0, filename.lastIndexOf("_") - 2)); - case 4: // Extended - String date = parts[2]; - String time = parts[3]; - return Optional.of(date + "_" + time.substring(0, 4)); - default: - return Optional.empty(); - } - } - - /** - * Removes all but after the last "/" - */ - public static String sanitizeFilename(String path) { - int i = path.lastIndexOf("/"); - if (i > 0) { - return path.substring(i + 1); - } else { - return path; - } - } - - public static String sanitizeFilename(Path path) { - return sanitizeFilename(path.toString()); - } - - - /** - * Converts a lscUtcTime format {@code UTC_yyyy-MM-dd_hh:mm} datetime to a {@link LocalDateTime} - * {@link LocalDateTime} is used, due to its definition of 'local time in ISO-8601 standard without time zone info'. - * - * @param lscUtcFormat The LSC UTC Timestamp in the format {@code UTC_yyyy-MM-dd_hh:mm} - */ - public static LocalDateTime convertUtc(String lscUtcFormat) { - return LocalDateTime.ofEpochSecond(convert(lscUtcFormat, true, null), 0, ZoneOffset.UTC); - } - - /** - * Converts a lscUtcTime format {@code yyyy-MM-dd_hh:mm} datetime to a {@link ZonedDateTime} - * - * @param lscFormat The LSC UTC Timestamp in the format {@code yyyy-MM-dd_hh:mm} - * @param zone A zoneid in format {@code Area/Region}, see {@linkplain ZoneId} - * @return - */ - public static ZonedDateTime convertLocal(String lscFormat, String zone) { - final long epochSec = convert(lscFormat, false, zone); - return ZonedDateTime.ofInstant(Instant.ofEpochSecond(epochSec), ZoneId.of(zone)); // Zone is parseable, otherwise not reached here - } - - public static String extractPhaseOfDay(ZonedDateTime dateTime) { - final int hour = dateTime.getHour(); - String tag; - if (hour > 7 && hour < 11) { - tag = "MORNING"; - } else if (hour >= 11 && hour < 14) { - tag = "NOON"; - } else if (hour >= 14 && hour < 17) { - tag = "AFTERNOON"; - } else if (hour >= 17 && hour < 22) { - tag = "EVENING"; - } else { - tag = "NIGHT"; - } - return tag; - } - - public static int extractYear(LocalDateTime time) { - return time.getYear(); - } - - public static int extractDay(LocalDateTime time){ - return time.getDayOfMonth(); - } - - public static String extractMonth(LocalDateTime time) { - return time.getMonth().name(); - } - - public static int extractHour(LocalDateTime utc) { - return utc.getHour(); - } - - public static String extractHourStr(LocalDateTime utc){ - return String.valueOf(extractHour(utc)); - } - - public static String extractYearStr(LocalDateTime utc){ - return String.valueOf(extractYear(utc)); - } - - public static String extractDayStr(LocalDateTime utc){ - return String.valueOf(extractHour(utc)); - } - - private static String getLscFormat(boolean utc) { - return (utc ? LSC_UTC_PREFIX : "") + LSC_FORMAT; - } - - private static long convert(String lscFormat, boolean utc, String zone) { - if (StringUtils.isBlank(lscFormat) || "NULL".equalsIgnoreCase(lscFormat)) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: BLANK. Given: " + lscFormat); - } - if (utc && !lscFormat.startsWith("UTC")) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: START. Given: " + lscFormat); - } else if (!utc && StringUtils.isBlank(zone)) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: ZONEBLANK. Given: " + lscFormat); - } - ZoneId zoneId = null; - try { - if (!utc) { - zoneId = ZoneId.of(zone); - } - } catch (DateTimeException e) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: ZONE. Given: " + lscFormat, e); - } - final String[] parts = lscFormat.split("_"); // ["UTC", "yyyy-mm-dd", "hh:mm"] - final int nbParts = utc ? 3 : 2; - if (parts.length < nbParts) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: DATE. Given: " + lscFormat); - } - final int dateIdx = utc ? 1 : 0; - final String[] date = parts[dateIdx].split("-"); // ["yyyy", "mm", "dd"] - if (date.length < 3) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: TIME. Given: " + lscFormat); - } - final int timeIdx = utc ? 2 : 1; - final String[] time = parts[timeIdx].split(":"); // ["hh", "mm"] - if (time.length < 2) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: TIME. Given: " + lscFormat); - } - try { - final int year = Integer.parseInt(date[0]); - final int month = Integer.parseInt(date[1]); - final int day = Integer.parseInt(date[2]); - final int hour = Integer.parseInt(time[0]); - final int minute = Integer.parseInt(time[1]); - if (utc) { - return LocalDateTime.of(year, month, day, hour, minute).toEpochSecond(ZoneOffset.UTC); - } else { - return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zoneId).toEpochSecond(); - } - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: NUMBERFORMAT. Given: " + lscFormat, e); - } - } - - /** - * Parses a minuteId {@code YYYYMMDD_HH:mm} to a {@link LocalDateTime}. - * The result is to be treated as UTC timestamp - */ - public static LocalDateTime fromMinuteId(String minuteId) { - // YYYYMMDD_HHMM - // 0123456789012 - try { - final int year = Integer.parseInt(minuteId.substring(0, 4)); - final int month = Integer.parseInt(minuteId.substring(4, 6)); - final int day = Integer.parseInt(minuteId.substring(6, 8)); - final int hour = Integer.parseInt(minuteId.substring(9, 11)); - final int minute = Integer.parseInt(minuteId.substring(11)); - return LocalDateTime.of(year, month, day, hour, minute); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Cannot parse due to invalid number format in minuteId"); - } - } - - public static boolean isMetaBlank(String meta){ - return StringUtils.isBlank(meta) || "NULL".equalsIgnoreCase(meta); - } - - public static boolean isAnyMetaBlank(String... meta){ - for(String m : meta){ - if(isMetaBlank(m)){ - return true; - } - } - return false; - } - - - private void readMetadata() throws IOException, CsvException { - LOGGER.info("Reading metadata file..."); - headerlessMetaContents = readFile(root); - LOGGER.info("Read metadata file"); - } - - private void initLookup(HashMap minuteIdPathMap, Path root) throws IOException { - long start = System.nanoTime(); - Path file = root.resolve(CONCEPTS_FILE_NAME); - LOGGER.info("Initialising Lookup from {}", file); - List contents = Files.readAllLines(file, StandardCharsets.UTF_8); - contents.stream().forEach(s -> { - if (!s.startsWith(KEY_MINUTEID)) { - // Catch the header line, which is not important - String[] items = s.split(","); // Csv - String minuteId = items[CONCEPTS_MINUTEID_COL]; - String imagePath = cleanImagePath(items[CONCEPTS_IMAGEPATH_COL]); - LOGGER.trace("Adding to lookup: {} <-> {}", minuteId, imagePath); - minuteIdPathMap.put(minuteId, imagePath); - } - }); - long time = (System.nanoTime() - start) / 1000000L; - LOGGER.info("Successfully initialised the lookup with {} entries in {}ms", minuteIdPathMap.size(), time); - } - - private List readListOfFiles(Path root) throws IOException, CsvException { - Path file = root.resolve(LSC_REPORT_VALID_FILNE_NAME); - CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); - List lookupContents = csv.readAll(); - lookupContents.remove(0); // Remove header - return lookupContents; - } - - private void initMap(List listOfFiles){ - listOfFiles.stream().forEach(s ->{ - final String sanitizeFilename = sanitizeFilename(s[0]); - final Optional minuteId = filenameToMinuteId(sanitizeFilename); - minuteId.ifPresent(value -> filenameToMinuteIdMap.put(sanitizeFilename, value)); - }); - } - - public void initMetadata() throws IOException, CsvException { - LOGGER.info("Initialising metadata lookup..."); - readMetadata(); - initMetaPerMinuteId(); - LOGGER.info("Initialised metadata lookup."); - } - - private void initMetaPerMinuteId(){ - headerlessMetaContents.stream().forEach(items ->{ - metaPerMinute.put(items[META_MIN_COL], items); - }); - } - - private List readFile(Path root) throws IOException, CsvException { - Path file = root.resolve(METADATA_FILE_NAME); - - CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); - List metadataFileContents = csv.readAll(); - metadataFileContents.remove(0); // remove header - return metadataFileContents; - } - - /** - * Immutable - * - * @return - */ - @Deprecated - public List getHeaderlessMetaContents() { - return Collections.unmodifiableList(headerlessMetaContents); - } - - public Map getMetaPerMinuteId(){ - return Collections.unmodifiableMap(metaPerMinute); - } - - public Map getFilenameToMinuteIdLookUp(){ - return Collections.unmodifiableMap(filenameToMinuteIdMap); - } - - /** - * Immutable - * - * @return - */ - @Deprecated - public Map getMinuteIdPathMap() { - return Collections.unmodifiableMap(minuteIdPathMap); - } + public static final int META_MIN_COL = 0; + public static final String META_MIN_NAME = "minute_id"; + public static final int META_UTC_COL = 1; + public static final String META_UTC_NAME = "utc_time"; + public static final int META_LOCAL_COL = 2; + public static final String META_LOCAL_NAME = "local_time"; + public static final int META_TIMEZONE_COL = 3; + public static final String META_TIMEZONE_NAME = "timezone"; + public static final int META_LAT_COL = 4; + public static final String META_LAT_NAME = "lat"; + public static final int META_LON_COL = 5; + public static final String META_LON_NAME = "lon"; + public static final int META_SEMANTIC_COL = 6; + public static final String META_SEMANTIC_NAME = "semantic_name"; + public static final int META_ELEVATION_COL = 7; + public static final String META_ELEVATION_NAME = "elevation"; + public static final int META_SPEED_COL = 8; + public static final String META_SPEED_NAME = "speed"; + public static final int META_HEART_COL = 9; + public static final String META_HEART_NAME = "heart"; + public static final int META_CALORIES_COL = 10; + public static final String META_CALORIES_NAME = "calories"; + public static final int META_ACTIVITY_COL = 11; + public static final String META_ACTIVITY_NAME = "activity_type"; + public static final int META_STEPS_COL = 12; + public static final String META_STEPS_NAME = "steps"; + //minute_id,utc_time,local_time,timezone,lat,lon,semantic_name,elevation,speed,heart,calories,activity_type,steps + public static final String[] META_NAMES = new String[]{META_MIN_NAME, META_UTC_NAME, META_LOCAL_NAME, META_TIMEZONE_NAME, META_LAT_NAME, META_LON_NAME, META_SEMANTIC_NAME, META_ELEVATION_NAME, META_SPEED_NAME, META_HEART_NAME, META_CALORIES_NAME, META_ACTIVITY_NAME, META_STEPS_NAME + }; + public static final Set META_COLUMNS_IN_USE = new HashSet<>(Lists.newArrayList(META_MIN_COL, META_UTC_COL, META_LOCAL_COL, META_LAT_COL, META_LON_COL, META_TIMEZONE_COL, META_SEMANTIC_COL, META_ACTIVITY_COL)); + public static final String DEFAULT_DOMAIN = "LSCMETA"; + public static final String LOCATION_DOMAIN = "LOCATION"; + public static final String DATETIME_DOMAIN = "TIME"; + public static final String LSC_UTC_PREFIX = "UTC_"; + public static final String LSC_FORMAT = "yyyy-MM-dd_hh:mm"; + + public static final String PROCESSED_META_UTC = "p_utc_standard"; + public static final String PROCESSED_META_LOCAL = "p_local_standard"; + public static final String PROCESSED_META_DATETIME = "p_datetime"; // based on filename, utc + public static final String PROCESSED_META_DAY_OF_WEEK = "p_day_of_week"; + public static final String PROCESSED_META_PHASE_OF_DAY = "p_phase_of_day"; + public static final String PROCESSED_META_HOUR_OF_DAY = "p_hour"; + public static final String PROCESSED_META_MONTH = "p_month"; + public static final String PROCESSED_META_YEAR = "p_year"; + public static final String PROCESSED_META_DAY = "p_day"; + + + private static final Logger LOGGER = LogManager.getLogger(LSCUtilities.class); + + + private static LSCUtilities instance = null; + private final Path root; + private List headerlessMetaContents; + + @Deprecated + private HashMap minuteIdPathMap = new HashMap<>(); + + private HashMap filenameToMinuteIdMap = new HashMap<>(); + private HashMap metaPerMinute = new HashMap<>(); + + private LSCUtilities(final Path root) throws IOException, CsvException { + this.root = root; + // this.initLookup(minuteIdPathMap, root); + LOGGER.info("Initialising lookup..."); + this.initMap(this.readListOfFiles(root)); + LOGGER.info("Initialisation done."); + // headerlessMetaContents = readFile(root); + } + + public static LSCUtilities create(final Path root) throws IOException, CsvException { + if (instance == null) { + instance = new LSCUtilities(root); + } else if (!root.equals(instance.root)) { + return instance; + } else { + LOGGER.warn("Instance already created with path {}. Returning this one", instance.root); + } + return instance; + } + + public static LSCUtilities getInstance() { + if (instance != null) { + return instance; + } else { + throw new IllegalStateException("There's no instance, use LSCUtilities.create first"); + } + } + + /** + * Writes the list as individual lines in a textfile. The file must already be created + * + * @param directory The directory the file resides in + * @param file The file to write into + * @param lines The lines to write + * @throws IOException If something goes wrong. Possibly due to the file not being created beforehand + */ + public static void writeLines(Path directory, String file, List lines) throws IOException { + final Path p = directory.resolve(file); + Files.write(p, lines, StandardOpenOption.TRUNCATE_EXISTING); + } + + /** + * Converts an imagepath to a corresponding segmentId. + *
    + * In the LSC2020 dataset, image paths (from file lsc2020-visual-concepts.csv) do contain more than the actual (image) file, hence some conversion is required + *
    + * Prepends is_, removes anything before a slash ("/"), if present, and after a dot (".") (i.e., file extension), if present + */ + public static String pathToSegmentId(String path) { + final int beginIdx = path.contains("/") ? path.lastIndexOf("/") + 1 : 0; + final int endIdx = path.contains(".") ? path.lastIndexOf(".") : path.length(); + final String prefix = path.startsWith("is_") ? "" : "is_"; + return prefix + path.substring(beginIdx, endIdx); + } + + public static String cleanImagePath(String path) { + if (path.startsWith("DATASETS/LSC2020/")) { + return path.substring("DATASETS/LSC2020/".length() + 1); + } else { + return path; + } + } + + /** + * Two potential naming formats:
    Simple Format: {@code YYYMMDD_HHMMSS_000.ext} and
    Extended Format: {@code prefix001_prefix_YYYMMDD_HHMMSS*.ext} Extension is jpg + *

    + * Example of Simple: {@code 20160914_192303_000.jpg}
    Example of Extended: {@code B00006542_21I6X0_20180505_140053E.JPG} + * + *

    + */ + public static Optional filenameToMinuteId(String filename) { + String[] parts = filename.split("_"); + switch (parts.length) { + case 3: // simple + return Optional.of(filename.substring(0, filename.lastIndexOf("_") - 2)); + case 4: // Extended + String date = parts[2]; + String time = parts[3]; + return Optional.of(date + "_" + time.substring(0, 4)); + default: + return Optional.empty(); + } + } + + /** + * Removes all but after the last "/" + */ + public static String sanitizeFilename(String path) { + int i = path.lastIndexOf("/"); + if (i > 0) { + return path.substring(i + 1); + } else { + return path; + } + } + + public static String sanitizeFilename(Path path) { + return sanitizeFilename(path.toString()); + } + + + /** + * Converts a lscUtcTime format {@code UTC_yyyy-MM-dd_hh:mm} datetime to a {@link LocalDateTime} {@link LocalDateTime} is used, due to its definition of 'local time in ISO-8601 standard without time zone info'. + * + * @param lscUtcFormat The LSC UTC Timestamp in the format {@code UTC_yyyy-MM-dd_hh:mm} + */ + public static LocalDateTime convertUtc(String lscUtcFormat) { + return LocalDateTime.ofEpochSecond(convert(lscUtcFormat, true, null), 0, ZoneOffset.UTC); + } + + /** + * Converts a lscUtcTime format {@code yyyy-MM-dd_hh:mm} datetime to a {@link ZonedDateTime} + * + * @param lscFormat The LSC UTC Timestamp in the format {@code yyyy-MM-dd_hh:mm} + * @param zone A zoneid in format {@code Area/Region}, see {@linkplain ZoneId} + * @return + */ + public static ZonedDateTime convertLocal(String lscFormat, String zone) { + final long epochSec = convert(lscFormat, false, zone); + return ZonedDateTime.ofInstant(Instant.ofEpochSecond(epochSec), ZoneId.of(zone)); // Zone is parseable, otherwise not reached here + } + + public static String extractPhaseOfDay(ZonedDateTime dateTime) { + final int hour = dateTime.getHour(); + String tag; + if (hour > 7 && hour < 11) { + tag = "MORNING"; + } else if (hour >= 11 && hour < 14) { + tag = "NOON"; + } else if (hour >= 14 && hour < 17) { + tag = "AFTERNOON"; + } else if (hour >= 17 && hour < 22) { + tag = "EVENING"; + } else { + tag = "NIGHT"; + } + return tag; + } + + public static int extractYear(LocalDateTime time) { + return time.getYear(); + } + + public static int extractDay(LocalDateTime time) { + return time.getDayOfMonth(); + } + + public static String extractMonth(LocalDateTime time) { + return time.getMonth().name(); + } + + public static int extractHour(LocalDateTime utc) { + return utc.getHour(); + } + + public static String extractHourStr(LocalDateTime utc) { + return String.valueOf(extractHour(utc)); + } + + public static String extractYearStr(LocalDateTime utc) { + return String.valueOf(extractYear(utc)); + } + + public static String extractDayStr(LocalDateTime utc) { + return String.valueOf(extractHour(utc)); + } + + private static String getLscFormat(boolean utc) { + return (utc ? LSC_UTC_PREFIX : "") + LSC_FORMAT; + } + + private static long convert(String lscFormat, boolean utc, String zone) { + if (StringUtils.isBlank(lscFormat) || "NULL".equalsIgnoreCase(lscFormat)) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: BLANK. Given: " + lscFormat); + } + if (utc && !lscFormat.startsWith("UTC")) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: START. Given: " + lscFormat); + } else if (!utc && StringUtils.isBlank(zone)) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: ZONEBLANK. Given: " + lscFormat); + } + ZoneId zoneId = null; + try { + if (!utc) { + zoneId = ZoneId.of(zone); + } + } catch (DateTimeException e) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: ZONE. Given: " + lscFormat, e); + } + final String[] parts = lscFormat.split("_"); // ["UTC", "yyyy-mm-dd", "hh:mm"] + final int nbParts = utc ? 3 : 2; + if (parts.length < nbParts) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: DATE. Given: " + lscFormat); + } + final int dateIdx = utc ? 1 : 0; + final String[] date = parts[dateIdx].split("-"); // ["yyyy", "mm", "dd"] + if (date.length < 3) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: TIME. Given: " + lscFormat); + } + final int timeIdx = utc ? 2 : 1; + final String[] time = parts[timeIdx].split(":"); // ["hh", "mm"] + if (time.length < 2) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: TIME. Given: " + lscFormat); + } + try { + final int year = Integer.parseInt(date[0]); + final int month = Integer.parseInt(date[1]); + final int day = Integer.parseInt(date[2]); + final int hour = Integer.parseInt(time[0]); + final int minute = Integer.parseInt(time[1]); + if (utc) { + return LocalDateTime.of(year, month, day, hour, minute).toEpochSecond(ZoneOffset.UTC); + } else { + return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zoneId).toEpochSecond(); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Cannot parse due to not in lscUtcFormat. Expected format: " + getLscFormat(utc) + ". Reason: NUMBERFORMAT. Given: " + lscFormat, e); + } + } + + /** + * Parses a minuteId {@code YYYYMMDD_HH:mm} to a {@link LocalDateTime}. The result is to be treated as UTC timestamp + */ + public static LocalDateTime fromMinuteId(String minuteId) { + // YYYYMMDD_HHMM + // 0123456789012 + try { + final int year = Integer.parseInt(minuteId.substring(0, 4)); + final int month = Integer.parseInt(minuteId.substring(4, 6)); + final int day = Integer.parseInt(minuteId.substring(6, 8)); + final int hour = Integer.parseInt(minuteId.substring(9, 11)); + final int minute = Integer.parseInt(minuteId.substring(11)); + return LocalDateTime.of(year, month, day, hour, minute); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Cannot parse due to invalid number format in minuteId"); + } + } + + public static boolean isMetaBlank(String meta) { + return StringUtils.isBlank(meta) || "NULL".equalsIgnoreCase(meta); + } + + public static boolean isAnyMetaBlank(String... meta) { + for (String m : meta) { + if (isMetaBlank(m)) { + return true; + } + } + return false; + } + + + private void readMetadata() throws IOException, CsvException { + LOGGER.info("Reading metadata file..."); + headerlessMetaContents = readFile(root); + LOGGER.info("Read metadata file"); + } + + private void initLookup(HashMap minuteIdPathMap, Path root) throws IOException { + long start = System.nanoTime(); + Path file = root.resolve(CONCEPTS_FILE_NAME); + LOGGER.info("Initialising Lookup from {}", file); + List contents = Files.readAllLines(file, StandardCharsets.UTF_8); + contents.stream().forEach(s -> { + if (!s.startsWith(KEY_MINUTEID)) { + // Catch the header line, which is not important + String[] items = s.split(","); // Csv + String minuteId = items[CONCEPTS_MINUTEID_COL]; + String imagePath = cleanImagePath(items[CONCEPTS_IMAGEPATH_COL]); + LOGGER.trace("Adding to lookup: {} <-> {}", minuteId, imagePath); + minuteIdPathMap.put(minuteId, imagePath); + } + }); + long time = (System.nanoTime() - start) / 1000000L; + LOGGER.info("Successfully initialised the lookup with {} entries in {}ms", minuteIdPathMap.size(), time); + } + + private List readListOfFiles(Path root) throws IOException, CsvException { + Path file = root.resolve(LSC_REPORT_VALID_FILNE_NAME); + CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); + List lookupContents = csv.readAll(); + lookupContents.remove(0); // Remove header + return lookupContents; + } + + private void initMap(List listOfFiles) { + listOfFiles.stream().forEach(s -> { + final String sanitizeFilename = sanitizeFilename(s[0]); + final Optional minuteId = filenameToMinuteId(sanitizeFilename); + minuteId.ifPresent(value -> filenameToMinuteIdMap.put(sanitizeFilename, value)); + }); + } + + public void initMetadata() throws IOException, CsvException { + LOGGER.info("Initialising metadata lookup..."); + readMetadata(); + initMetaPerMinuteId(); + LOGGER.info("Initialised metadata lookup."); + } + + private void initMetaPerMinuteId() { + headerlessMetaContents.stream().forEach(items -> { + metaPerMinute.put(items[META_MIN_COL], items); + }); + } + + private List readFile(Path root) throws IOException, CsvException { + Path file = root.resolve(METADATA_FILE_NAME); + + CSVReader csv = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); + List metadataFileContents = csv.readAll(); + metadataFileContents.remove(0); // remove header + return metadataFileContents; + } + + /** + * Immutable + * + * @return + */ + @Deprecated + public List getHeaderlessMetaContents() { + return Collections.unmodifiableList(headerlessMetaContents); + } + + public Map getMetaPerMinuteId() { + return Collections.unmodifiableMap(metaPerMinute); + } + + public Map getFilenameToMinuteIdLookUp() { + return Collections.unmodifiableMap(filenameToMinuteIdMap); + } + + /** + * Immutable + * + * @return + */ + @Deprecated + public Map getMinuteIdPathMap() { + return Collections.unmodifiableMap(minuteIdPathMap); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImportHandler.java index 44a1b3b29..692f5c694 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImportHandler.java @@ -1,6 +1,8 @@ package org.vitrivr.cineast.standalone.importer.lsc2020; import com.opencsv.exceptions.CsvException; +import java.io.IOException; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor; @@ -8,40 +10,37 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.io.IOException; -import java.nio.file.Path; - public class MetaImportHandler extends DataImportHandler { - private static final Logger LOGGER = LogManager.getLogger(MetaImportHandler.class); + private static final Logger LOGGER = LogManager.getLogger(MetaImportHandler.class); - private final boolean clean; + private final boolean clean; - public MetaImportHandler(int threads, int batchSize, boolean clean){ - super(threads, batchSize); - this.clean= clean; - final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); - /* Beware, this drops the table */ - if (clean) { - LOGGER.info("Dropping table ..."); - ec.dropSegmentMetadataEntity(); - LOGGER.info("Finished dropping table for entity "); - ec.createSegmentMetadataEntity(); - LOGGER.info("Re-created SegmentMetaData entity"); - ec.close(); - } + public MetaImportHandler(int threads, int batchSize, boolean clean) { + super(threads, batchSize); + this.clean = clean; + final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); + /* Beware, this drops the table */ + if (clean) { + LOGGER.info("Dropping table ..."); + ec.dropSegmentMetadataEntity(); + LOGGER.info("Finished dropping table for entity "); + ec.createSegmentMetadataEntity(); + LOGGER.info("Re-created SegmentMetaData entity"); + ec.close(); } + } - @Override - public void doImport(Path path) { - try { - LSCUtilities.create(path); - } catch (IOException | CsvException e) { - LOGGER.fatal("Error in initialisation", e); - LOGGER.fatal("Crashing now"); - return; - } - LOGGER.info("Starting LSC metadata import from folder {}", path); - this.futures.add(this.service.submit(new DataImportRunner(new MetaImporter(path), MediaSegmentMetadataDescriptor.ENTITY, "lsc-metadata", clean))); + @Override + public void doImport(Path path) { + try { + LSCUtilities.create(path); + } catch (IOException | CsvException e) { + LOGGER.fatal("Error in initialisation", e); + LOGGER.fatal("Crashing now"); + return; } + LOGGER.info("Starting LSC metadata import from folder {}", path); + this.futures.add(this.service.submit(new DataImportRunner(new MetaImporter(path), MediaSegmentMetadataDescriptor.ENTITY, "lsc-metadata", clean))); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java index cbc218a6b..b01dfc2ef 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MetaImporter.java @@ -50,7 +50,6 @@ public MetaImporter(Path path) { throw new RuntimeException("Failed to prepare metadata readout", e); } - createLogFiles(); LOGGER.info("Finished setup of Importer. Importing now..."); } @@ -182,7 +181,7 @@ private void writeLogs() { if (!logsEnabled) { return; } - if (nextsSinceLastDump++> logCadence) { + if (nextsSinceLastDump++ > logCadence) { try { writeLines(LSCUtilities.META_NO_PATH_FILE, metaNoPath); writeLines(LSCUtilities.WRITTEN_FILE, written); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImportHandler.java index 517ffc0de..89d411803 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImportHandler.java @@ -6,13 +6,12 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.features.SegmentTags; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import org.vitrivr.cineast.standalone.importer.lsc2020.MyscealTagImporter; public class MyscealTagImportHandler extends DataImportHandler { private static final Logger LOGGER = LogManager.getLogger(); - public MyscealTagImportHandler(int threads, int batchsize){ + public MyscealTagImportHandler(int threads, int batchsize) { super(threads, batchsize); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java index 92b70029c..e0bef9f41 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/MyscealTagImporter.java @@ -32,8 +32,7 @@ public class MyscealTagImporter implements Importer readNext() { - if(this.tagLookup){ + if (this.tagLookup) { return readNextTagLookup(); - }else { + } else { return readNextTagScore(); } } /** - * Returns the next tag score element. - * In other words, this returns the next triple segmentId, tagId, score als long as there are these triples. - * Those triples are constructed by first getting the current segmentId and then iterating over this segment's tags until they are all processed + * Returns the next tag score element. In other words, this returns the next triple segmentId, tagId, score als long as there are these triples. Those triples are constructed by first getting the current segmentId and then iterating over this segment's tags until they are all processed */ private Map readNextTagScore() { - do{ - if(currentSegmentId == null && tagScoreIterator.hasNext()){ + do { + if (currentSegmentId == null && tagScoreIterator.hasNext()) { /* Get current segment id and iterator */ - final Map.Entry>> entry = tagScoreIterator.next(); + final Map.Entry>> entry = tagScoreIterator.next(); currentSegmentId = entry.getKey(); currentIterator = entry.getValue().iterator(); } - if(currentIterator.hasNext()){ + if (currentIterator.hasNext()) { /* Commit current segment tag with score */ final Pair segmentTag = currentIterator.next(); final Map map = new HashMap<>(); @@ -82,12 +79,12 @@ private Map readNextTagScore() { map.put("tagid", PrimitiveTypeProvider.fromObject(segmentTag.first)); map.put("score", PrimitiveTypeProvider.fromObject(segmentTag.second)); return map; - }else{ + } else { /* Reset current iterator & segmentId */ currentIterator = null; currentSegmentId = null; } - }while(currentIterator == null && tagScoreIterator.hasNext()); + } while (currentIterator == null && tagScoreIterator.hasNext()); return null; } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImportHandler.java index f18e47f82..5b5322719 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImportHandler.java @@ -1,42 +1,38 @@ package org.vitrivr.cineast.standalone.importer.lsc2020; import com.opencsv.exceptions.CsvException; +import java.io.IOException; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; -import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.OCRSearch; -import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.io.IOException; -import java.nio.file.Path; - public class OCRImportHandler extends DataImportHandler { - private static final Logger LOGGER = LogManager.getLogger(OCRImportHandler.class); - private final boolean clean; + private static final Logger LOGGER = LogManager.getLogger(OCRImportHandler.class); + private final boolean clean; - /** - * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. - * - * @param threads Number of threads to use for data import. - * @param batchsize Size of data batches that are sent to the persistence layer. - */ - public OCRImportHandler(int threads, int batchsize, boolean clean) { - super(threads, batchsize); - this.clean = clean; - } + /** + * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. + * + * @param threads Number of threads to use for data import. + * @param batchsize Size of data batches that are sent to the persistence layer. + */ + public OCRImportHandler(int threads, int batchsize, boolean clean) { + super(threads, batchsize); + this.clean = clean; + } - @Override - public void doImport(Path path) { - try { - LSCUtilities.create(path); - } catch (IOException | CsvException e) { - LOGGER.fatal("Error in init", e); - return; - } - LOGGER.info("Starting LSC OCR import from {}", path); - this.futures.add(this.service.submit(new DataImportRunner(new OCRImporter(path), OCRSearch.OCR_TABLE_NAME, "lsc-ocr", clean))); + @Override + public void doImport(Path path) { + try { + LSCUtilities.create(path); + } catch (IOException | CsvException e) { + LOGGER.fatal("Error in init", e); + return; } + LOGGER.info("Starting LSC OCR import from {}", path); + this.futures.add(this.service.submit(new DataImportRunner(new OCRImporter(path), OCRSearch.OCR_TABLE_NAME, "lsc-ocr", clean))); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImporter.java index c91f3e122..6d0cd3ede 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/OCRImporter.java @@ -2,18 +2,17 @@ import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; /** * Headerless csv file. column 0 is filename, column 1 is OCR @@ -21,41 +20,41 @@ */ public class OCRImporter implements Importer> { - private static final Logger LOGGER = LogManager.getLogger(OCRImporter.class); - - private final Path root; - - private Iterator> iterator; - - public OCRImporter(Path root) { - this.root = root; - try { - CSVReader reader = new CSVReader(Files.newBufferedReader(root.resolve("OCR_result.csv"))); - iterator = reader.readAll().stream().map(line -> { - String segment = LSCUtilities.pathToSegmentId(line[0]); - PrimitiveTypeProvider value = PrimitiveTypeProvider.fromObject(line[1]); - HashMap map = new HashMap<>(2); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(segment)); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], value); - return (Map)map; // Apparently, without explicit casting the compiler is a sad panda - }).iterator(); - LOGGER.info("Successfully read and parsed the import file"); - } catch (IOException | CsvException e) { - LOGGER.fatal("Could not read importfile", e); - } + private static final Logger LOGGER = LogManager.getLogger(OCRImporter.class); + + private final Path root; + + private Iterator> iterator; + + public OCRImporter(Path root) { + this.root = root; + try { + CSVReader reader = new CSVReader(Files.newBufferedReader(root.resolve("OCR_result.csv"))); + iterator = reader.readAll().stream().map(line -> { + String segment = LSCUtilities.pathToSegmentId(line[0]); + PrimitiveTypeProvider value = PrimitiveTypeProvider.fromObject(line[1]); + HashMap map = new HashMap<>(2); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(segment)); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], value); + return (Map) map; // Apparently, without explicit casting the compiler is a sad panda + }).iterator(); + LOGGER.info("Successfully read and parsed the import file"); + } catch (IOException | CsvException e) { + LOGGER.fatal("Could not read importfile", e); } - - @Override - public Map readNext() { - if(iterator != null && iterator.hasNext()){ - return iterator.next(); - }else{ - return null; - } + } + + @Override + public Map readNext() { + if (iterator != null && iterator.hasNext()) { + return iterator.next(); + } else { + return null; } + } - @Override - public Map convert(Map data) { - return data; - } + @Override + public Map convert(Map data) { + return data; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImportHandler.java index 720c55dcc..cfec16e86 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImportHandler.java @@ -67,8 +67,8 @@ public ProcessingMetaImportHandler(int threads, int batchsize, Mode mode, boolea } else { metaAsTable = false; } - if(clean){ - createEntityForced(); + if (clean) { + createEntityForced(); } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java index 674060d1a..3f03d853e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/ProcessingMetaImporter.java @@ -1,5 +1,16 @@ package org.vitrivr.cineast.standalone.importer.lsc2020; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,328 +18,323 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.importer.Importer; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZonedDateTime; -import java.util.*; - public class ProcessingMetaImporter implements Importer> { - public static final List TAG_CANDIDATES = Arrays.asList(LSCUtilities.META_SEMANTIC_COL, LSCUtilities.META_ACTIVITY_COL, LSCUtilities.META_TIMEZONE_COL, LSCUtilities.META_UTC_COL, LSCUtilities.META_LOCAL_COL); - private static final Logger LOGGER = LogManager.getLogger(ProcessingMetaImporter.class); - private final Type type; - private final Map metaPerMinuteId; - private final Map filenameToMinuteIdLookUp; - private Iterator> currentDataIterator = null; - private Iterator> iterator; - private Map minuteIdPathMap = new HashMap<>(); - private HashSet uniqueList = new HashSet<>(); - private HashSet uniqueTemporalMetadata = new HashSet<>(); + public static final List TAG_CANDIDATES = Arrays.asList(LSCUtilities.META_SEMANTIC_COL, LSCUtilities.META_ACTIVITY_COL, LSCUtilities.META_TIMEZONE_COL, LSCUtilities.META_UTC_COL, LSCUtilities.META_LOCAL_COL); + private static final Logger LOGGER = LogManager.getLogger(ProcessingMetaImporter.class); + private final Type type; + private final Map metaPerMinuteId; + private final Map filenameToMinuteIdLookUp; + private Iterator> currentDataIterator = null; + private Iterator> iterator; + private Map minuteIdPathMap = new HashMap<>(); + private HashSet uniqueList = new HashSet<>(); + private HashSet uniqueTemporalMetadata = new HashSet<>(); - public ProcessingMetaImporter(final Path path, final Type type) { - this.type = type; + public ProcessingMetaImporter(final Path path, final Type type) { + this.type = type; - // We do not initialise LSCUtilities, as they are created outside. - LSCUtilities lsc = LSCUtilities.getInstance(); - filenameToMinuteIdLookUp = lsc.getFilenameToMinuteIdLookUp(); - metaPerMinuteId = lsc.getMetaPerMinuteId(); - iterator = filenameToMinuteIdLookUp.entrySet().iterator(); - LOGGER.info("Finished setup of Importer. Importing now..."); - } + // We do not initialise LSCUtilities, as they are created outside. + LSCUtilities lsc = LSCUtilities.getInstance(); + filenameToMinuteIdLookUp = lsc.getFilenameToMinuteIdLookUp(); + metaPerMinuteId = lsc.getMetaPerMinuteId(); + iterator = filenameToMinuteIdLookUp.entrySet().iterator(); + LOGGER.info("Finished setup of Importer. Importing now..."); + } - private Optional>> parseLine(String path, String[] items) { - String minuteId = items[LSCUtilities.META_MIN_COL]; - if (path == null) { - // cannot resolve - return Optional.empty(); - } - String segmentId = LSCUtilities.cleanImagePath(path); - segmentId = LSCUtilities.pathToSegmentId(segmentId); - List> list = new ArrayList<>(); - switch (this.type) { - case TAG: - case TAG_LOOKUP: - Optional>> maps = parseAsTag(segmentId, path, items); - maps.ifPresent(list::addAll); + private Optional>> parseLine(String path, String[] items) { + String minuteId = items[LSCUtilities.META_MIN_COL]; + if (path == null) { + // cannot resolve + return Optional.empty(); + } + String segmentId = LSCUtilities.cleanImagePath(path); + segmentId = LSCUtilities.pathToSegmentId(segmentId); + List> list = new ArrayList<>(); + switch (this.type) { + case TAG: + case TAG_LOOKUP: + Optional>> maps = parseAsTag(segmentId, path, items); + maps.ifPresent(list::addAll); - break; - case META_AS_TABLE: - list.add(parseAsMeta(segmentId, path, items)); - break; - } - return Optional.of(list); + break; + case META_AS_TABLE: + list.add(parseAsMeta(segmentId, path, items)); + break; } + return Optional.of(list); + } - private Map parseAsMeta(String segmentId, String path, String[] items) { - final HashMap map = new HashMap<>(items.length + 9); + private Map parseAsMeta(String segmentId, String path, String[] items) { + final HashMap map = new HashMap<>(items.length + 9); - map.put("id", PrimitiveTypeProvider.fromObject(segmentId)); - for (int i = 0; i < items.length; i++) { - switch (i) { - case LSCUtilities.META_LAT_COL: - case LSCUtilities.META_LON_COL: - if(StringUtils.isNotBlank(items[i]) && !items[i].equalsIgnoreCase("null")){ - // Only add if useful + map.put("id", PrimitiveTypeProvider.fromObject(segmentId)); + for (int i = 0; i < items.length; i++) { + switch (i) { + case LSCUtilities.META_LAT_COL: + case LSCUtilities.META_LON_COL: + if (StringUtils.isNotBlank(items[i]) && !items[i].equalsIgnoreCase("null")) { + // Only add if useful - map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(Double.parseDouble(items[i]))); - }else{ - LOGGER.trace("Did not include "+segmentId+" :"+LSCUtilities.META_NAMES[i]+", beacuse its blank or null:"+ items[i]); - map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(Double.NaN)); - } - break; - default: - map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(items[i])); - break; - } - } - final String minuteId = LSCUtilities.filenameToMinuteId(path).get(); // Purposely no isPresent check, because it should not be possible - final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId); - map.put(LSCUtilities.PROCESSED_META_DATETIME, PrimitiveTypeProvider.fromObject(dt.toString())); - map.put(LSCUtilities.PROCESSED_META_DAY_OF_WEEK, PrimitiveTypeProvider.fromObject(String.valueOf(dt.getDayOfWeek()))); - map.put(LSCUtilities.PROCESSED_META_MONTH, PrimitiveTypeProvider.fromObject(LSCUtilities.extractMonth(dt))); - // Numeric values - map.put(LSCUtilities.PROCESSED_META_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractDay(dt))); - map.put(LSCUtilities.PROCESSED_META_YEAR, PrimitiveTypeProvider.fromObject(LSCUtilities.extractYear(dt))); - map.put(LSCUtilities.PROCESSED_META_HOUR_OF_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractHour(dt))); - if (StringUtils.isNotBlank(items[LSCUtilities.META_UTC_COL]) && !items[LSCUtilities.META_UTC_COL].equalsIgnoreCase("null")) { - map.put(LSCUtilities.PROCESSED_META_UTC, PrimitiveTypeProvider.fromObject(LSCUtilities.convertUtc(items[LSCUtilities.META_UTC_COL]).toString())); - } - // Processed temporal metadata, based on metadata file - if ((StringUtils.isNotBlank(items[LSCUtilities.META_LOCAL_COL]) && !items[LSCUtilities.META_LOCAL_COL].equalsIgnoreCase("null")) - && (StringUtils.isNotBlank(items[LSCUtilities.META_TIMEZONE_COL]) && !items[LSCUtilities.META_TIMEZONE_COL].equalsIgnoreCase("null"))) { - final ZonedDateTime zdt = LSCUtilities.convertLocal(items[LSCUtilities.META_LOCAL_COL], items[LSCUtilities.META_TIMEZONE_COL]); - map.put(LSCUtilities.PROCESSED_META_LOCAL, PrimitiveTypeProvider.fromObject(zdt.toString())); - map.put(LSCUtilities.PROCESSED_META_PHASE_OF_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractPhaseOfDay(zdt))); - }else{ - map.put(LSCUtilities.PROCESSED_META_LOCAL, PrimitiveTypeProvider.fromObject(null)); - map.put(LSCUtilities.PROCESSED_META_PHASE_OF_DAY, PrimitiveTypeProvider.fromObject(null)); - } - - return map; + map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(Double.parseDouble(items[i]))); + } else { + LOGGER.trace("Did not include " + segmentId + " :" + LSCUtilities.META_NAMES[i] + ", beacuse its blank or null:" + items[i]); + map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(Double.NaN)); + } + break; + default: + map.put(LSCUtilities.META_NAMES[i], PrimitiveTypeProvider.fromObject(items[i])); + break; + } + } + final String minuteId = LSCUtilities.filenameToMinuteId(path).get(); // Purposely no isPresent check, because it should not be possible + final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId); + map.put(LSCUtilities.PROCESSED_META_DATETIME, PrimitiveTypeProvider.fromObject(dt.toString())); + map.put(LSCUtilities.PROCESSED_META_DAY_OF_WEEK, PrimitiveTypeProvider.fromObject(String.valueOf(dt.getDayOfWeek()))); + map.put(LSCUtilities.PROCESSED_META_MONTH, PrimitiveTypeProvider.fromObject(LSCUtilities.extractMonth(dt))); + // Numeric values + map.put(LSCUtilities.PROCESSED_META_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractDay(dt))); + map.put(LSCUtilities.PROCESSED_META_YEAR, PrimitiveTypeProvider.fromObject(LSCUtilities.extractYear(dt))); + map.put(LSCUtilities.PROCESSED_META_HOUR_OF_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractHour(dt))); + if (StringUtils.isNotBlank(items[LSCUtilities.META_UTC_COL]) && !items[LSCUtilities.META_UTC_COL].equalsIgnoreCase("null")) { + map.put(LSCUtilities.PROCESSED_META_UTC, PrimitiveTypeProvider.fromObject(LSCUtilities.convertUtc(items[LSCUtilities.META_UTC_COL]).toString())); + } + // Processed temporal metadata, based on metadata file + if ((StringUtils.isNotBlank(items[LSCUtilities.META_LOCAL_COL]) && !items[LSCUtilities.META_LOCAL_COL].equalsIgnoreCase("null")) + && (StringUtils.isNotBlank(items[LSCUtilities.META_TIMEZONE_COL]) && !items[LSCUtilities.META_TIMEZONE_COL].equalsIgnoreCase("null"))) { + final ZonedDateTime zdt = LSCUtilities.convertLocal(items[LSCUtilities.META_LOCAL_COL], items[LSCUtilities.META_TIMEZONE_COL]); + map.put(LSCUtilities.PROCESSED_META_LOCAL, PrimitiveTypeProvider.fromObject(zdt.toString())); + map.put(LSCUtilities.PROCESSED_META_PHASE_OF_DAY, PrimitiveTypeProvider.fromObject(LSCUtilities.extractPhaseOfDay(zdt))); + } else { + map.put(LSCUtilities.PROCESSED_META_LOCAL, PrimitiveTypeProvider.fromObject(null)); + map.put(LSCUtilities.PROCESSED_META_PHASE_OF_DAY, PrimitiveTypeProvider.fromObject(null)); } - private Optional>> parseAsTag(String segmentid, String path, String[] items) { - List> list = new ArrayList<>(); - for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) { - if (items[i].equalsIgnoreCase("null")) { - continue; - } - Optional> data = parse(segmentid, path, items, i); - if (!data.isPresent()) { - continue; - } else { - list.add(data.get()); - } - } - // Additionally processed meta as tags: DayOfWeek, Month, Day, Year, HourOfDay - final String minuteId = LSCUtilities.filenameToMinuteId(path).get(); // Purposely no isPresent check, because it should not be possible - final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId); - BiProducer> parser; - if (type == Type.TAG) { - parser = this::toTag; - } else if (type == Type.TAG_LOOKUP) { - parser = this::toTagLookup; - } else { - parser = null; // To willfully throw NPE - } - String dow = String.valueOf(dt.getDayOfWeek()); - String month = LSCUtilities.extractMonth(dt); - String year = LSCUtilities.extractYearStr(dt); - String day = LSCUtilities.extractDayStr(dt); - String hod = LSCUtilities.extractHourStr(dt); + return map; + } - // Day of week - if(onlyUnique()){ - // tag lookup, it must be unique - if(isUniqueTemporalContext(dow)){ - list.add(parser.produce(segmentid, dow)); - } - }else{ - list.add(parser.produce(segmentid, dow)); - } - // Month - if(onlyUnique()){ - if(isUniqueTemporalContext(month)){ - list.add(parser.produce(segmentid, month)); - } - }else{ - list.add(parser.produce(segmentid, month)); - } - // Year - if(onlyUnique()){ - if(isUniqueTemporalContext(year)){ - list.add(parser.produce(segmentid, year)); - } - }else{ - list.add(parser.produce(segmentid, year)); - } - // Day - if(onlyUnique()){ - if(isUniqueTemporalContext(day)){ - list.add(parser.produce(segmentid, day)); - } - }else{ - list.add(parser.produce(segmentid, day)); - } - // HourOfDay - if(onlyUnique()){ - if(isUniqueTemporalContext(hod)){ - list.add(parser.produce(segmentid, hod)); - } - }else{ - list.add(parser.produce(segmentid, hod)); - } - if (list.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(list); - } + private Optional>> parseAsTag(String segmentid, String path, String[] items) { + List> list = new ArrayList<>(); + for (int i = 0; i < LSCUtilities.META_NAMES.length; i++) { + if (items[i].equalsIgnoreCase("null")) { + continue; + } + Optional> data = parse(segmentid, path, items, i); + if (!data.isPresent()) { + continue; + } else { + list.add(data.get()); + } } - - private Optional> parse(String segmentid, String path, String[] items, int index) { - switch (this.type) { - case TAG: - return parseTag(segmentid, path, items, index); - case TAG_LOOKUP: - return parseTagForLookup(segmentid, path, items, index); - default: - return Optional.empty(); - } + // Additionally processed meta as tags: DayOfWeek, Month, Day, Year, HourOfDay + final String minuteId = LSCUtilities.filenameToMinuteId(path).get(); // Purposely no isPresent check, because it should not be possible + final LocalDateTime dt = LSCUtilities.fromMinuteId(minuteId); + BiProducer> parser; + if (type == Type.TAG) { + parser = this::toTag; + } else if (type == Type.TAG_LOOKUP) { + parser = this::toTagLookup; + } else { + parser = null; // To willfully throw NPE } + String dow = String.valueOf(dt.getDayOfWeek()); + String month = LSCUtilities.extractMonth(dt); + String year = LSCUtilities.extractYearStr(dt); + String day = LSCUtilities.extractDayStr(dt); + String hod = LSCUtilities.extractHourStr(dt); - private Optional> parseTag(String segmentid, String path, String[] items, int index) { - if (TAG_CANDIDATES.contains(index)) { - final String tag = metaAsTag(items, index); - return Optional.of(toTag(segmentid, tag)); - } else { - return Optional.empty(); - } + // Day of week + if (onlyUnique()) { + // tag lookup, it must be unique + if (isUniqueTemporalContext(dow)) { + list.add(parser.produce(segmentid, dow)); + } + } else { + list.add(parser.produce(segmentid, dow)); } - - private Map toTag(String segmentid, String tag) { - return toTag(segmentid, tag, 1); + // Month + if (onlyUnique()) { + if (isUniqueTemporalContext(month)) { + list.add(parser.produce(segmentid, month)); + } + } else { + list.add(parser.produce(segmentid, month)); } - - private Map toTag(String segmentid, String tag, int score) { - final HashMap map = new HashMap<>(3); - map.put("id", PrimitiveTypeProvider.fromObject(segmentid)); - map.put("tagid", PrimitiveTypeProvider.fromObject(tag)); - map.put("score", PrimitiveTypeProvider.fromObject(score)); - return map; + // Year + if (onlyUnique()) { + if (isUniqueTemporalContext(year)) { + list.add(parser.produce(segmentid, year)); + } + } else { + list.add(parser.produce(segmentid, year)); + } + // Day + if (onlyUnique()) { + if (isUniqueTemporalContext(day)) { + list.add(parser.produce(segmentid, day)); + } + } else { + list.add(parser.produce(segmentid, day)); } + // HourOfDay + if (onlyUnique()) { + if (isUniqueTemporalContext(hod)) { + list.add(parser.produce(segmentid, hod)); + } + } else { + list.add(parser.produce(segmentid, hod)); + } + if (list.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(list); + } + } - /** - * Must only be called with index values for valid tags - * tags: id === name, description is empty (no tag expansion done) - */ - private Optional> parseTagForLookup(String segmentid, String path, String[] items, int index) { - if (TAG_CANDIDATES.contains(index)) { - final String tag = metaAsTag(items, index); - if (onlyUnique()) { - if (!isUnique(tag)) { - return Optional.empty(); - } - } - return Optional.of(toTagLookup(segmentid, tag)); - } else { - return Optional.empty(); - } + private Optional> parse(String segmentid, String path, String[] items, int index) { + switch (this.type) { + case TAG: + return parseTag(segmentid, path, items, index); + case TAG_LOOKUP: + return parseTagForLookup(segmentid, path, items, index); + default: + return Optional.empty(); } + } - private Map toTagLookup(String segmentid, String tag) { - final HashMap map = new HashMap<>(3); - map.put(TagReader.TAG_ID_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); - map.put(TagReader.TAG_NAME_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); - map.put(TagReader.TAG_DESCRIPTION_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); // LSC Context no description of tags available. Use tag name as "description" - return map; + private Optional> parseTag(String segmentid, String path, String[] items, int index) { + if (TAG_CANDIDATES.contains(index)) { + final String tag = metaAsTag(items, index); + return Optional.of(toTag(segmentid, tag)); + } else { + return Optional.empty(); } + } + private Map toTag(String segmentid, String tag) { + return toTag(segmentid, tag, 1); + } - private String metaAsTag(String[] items, int index) { - String tag; - if (index == LSCUtilities.META_UTC_COL) { - tag = LSCUtilities.convertUtc(items[index]).getDayOfWeek().toString(); - } else if (index == LSCUtilities.META_LOCAL_COL) { - String zone = items[LSCUtilities.META_TIMEZONE_COL]; - tag = LSCUtilities.extractPhaseOfDay(LSCUtilities.convertLocal(items[index], zone)); - } else { - tag = items[index]; + private Map toTag(String segmentid, String tag, int score) { + final HashMap map = new HashMap<>(3); + map.put("id", PrimitiveTypeProvider.fromObject(segmentid)); + map.put("tagid", PrimitiveTypeProvider.fromObject(tag)); + map.put("score", PrimitiveTypeProvider.fromObject(score)); + return map; + } + + /** + * Must only be called with index values for valid tags tags: id === name, description is empty (no tag expansion done) + */ + private Optional> parseTagForLookup(String segmentid, String path, String[] items, int index) { + if (TAG_CANDIDATES.contains(index)) { + final String tag = metaAsTag(items, index); + if (onlyUnique()) { + if (!isUnique(tag)) { + return Optional.empty(); } - return tag; + } + return Optional.of(toTagLookup(segmentid, tag)); + } else { + return Optional.empty(); } + } - @Override - public Map readNext() { - // Has to read new line - do { - if (this.currentDataIterator == null && this.iterator.hasNext()) { - // LOGGER.trace("Init / Next: dataIt==null && it.hasNext"); - Map.Entry next = this.iterator.next(); - Optional>> parsed = parseLine(next.getKey(), metaPerMinuteId.get(next.getValue())); - parsed.ifPresent(maps -> this.currentDataIterator = maps.iterator()); - } - if (this.currentDataIterator != null && this.currentDataIterator.hasNext()) { - // LOGGER.trace("dataNext: dataIt.hasNext"); - Map out = this.currentDataIterator.next(); - if (!currentDataIterator.hasNext()) { - // reset, so Init / Next occurs - currentDataIterator = null; - } - // LOGGER.trace("Returning metadata: {}", out); - return out; - } - } while (this.currentDataIterator == null && this.iterator.hasNext()); - LOGGER.info("No more to read. Stopping"); - return null; - } + private Map toTagLookup(String segmentid, String tag) { + final HashMap map = new HashMap<>(3); + map.put(TagReader.TAG_ID_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); + map.put(TagReader.TAG_NAME_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); + map.put(TagReader.TAG_DESCRIPTION_COLUMNNAME, PrimitiveTypeProvider.fromObject(tag)); // LSC Context no description of tags available. Use tag name as "description" + return map; + } - /** - * Checks if the needle is in the unique set. If so, it's not unique. Otherwise, it's unique and added to the list - */ - private boolean isUnique(String needle) { - boolean found = uniqueList.contains(needle); - if (!found) { - uniqueList.add(needle); - } - return !found; + + private String metaAsTag(String[] items, int index) { + String tag; + if (index == LSCUtilities.META_UTC_COL) { + tag = LSCUtilities.convertUtc(items[index]).getDayOfWeek().toString(); + } else if (index == LSCUtilities.META_LOCAL_COL) { + String zone = items[LSCUtilities.META_TIMEZONE_COL]; + tag = LSCUtilities.extractPhaseOfDay(LSCUtilities.convertLocal(items[index], zone)); + } else { + tag = items[index]; } + return tag; + } - private boolean isUniqueTemporalContext(String temp){ - boolean found = uniqueTemporalMetadata.contains(temp); - if(!found){ - uniqueTemporalMetadata.add(temp); + @Override + public Map readNext() { + // Has to read new line + do { + if (this.currentDataIterator == null && this.iterator.hasNext()) { + // LOGGER.trace("Init / Next: dataIt==null && it.hasNext"); + Map.Entry next = this.iterator.next(); + Optional>> parsed = parseLine(next.getKey(), metaPerMinuteId.get(next.getValue())); + parsed.ifPresent(maps -> this.currentDataIterator = maps.iterator()); + } + if (this.currentDataIterator != null && this.currentDataIterator.hasNext()) { + // LOGGER.trace("dataNext: dataIt.hasNext"); + Map out = this.currentDataIterator.next(); + if (!currentDataIterator.hasNext()) { + // reset, so Init / Next occurs + currentDataIterator = null; } - return !found; + // LOGGER.trace("Returning metadata: {}", out); + return out; + } + } while (this.currentDataIterator == null && this.iterator.hasNext()); + LOGGER.info("No more to read. Stopping"); + return null; + } + + /** + * Checks if the needle is in the unique set. If so, it's not unique. Otherwise, it's unique and added to the list + */ + private boolean isUnique(String needle) { + boolean found = uniqueList.contains(needle); + if (!found) { + uniqueList.add(needle); } + return !found; + } - private boolean onlyUnique() { - return type == Type.TAG_LOOKUP; + private boolean isUniqueTemporalContext(String temp) { + boolean found = uniqueTemporalMetadata.contains(temp); + if (!found) { + uniqueTemporalMetadata.add(temp); } + return !found; + } + private boolean onlyUnique() { + return type == Type.TAG_LOOKUP; + } - @Override - public Map convert(Map data) { - return data; - } - public enum Type { - /** - * Tags for retrieval - */ - TAG, - /** - * Tags for tag lookup (i.e. autocomplete in vitrivr-ng - */ - TAG_LOOKUP, - /** - * Meta as is and certain processed metadata in a conventional table. - */ - META_AS_TABLE - } + @Override + public Map convert(Map data) { + return data; + } - @FunctionalInterface - public interface BiProducer { - O produce(I1 i1, I2 i2); - } + public enum Type { + /** + * Tags for retrieval + */ + TAG, + /** + * Tags for tag lookup (i.e. autocomplete in vitrivr-ng + */ + TAG_LOOKUP, + /** + * Meta as is and certain processed metadata in a conventional table. + */ + META_AS_TABLE + } + + @FunctionalInterface + public interface BiProducer { + + O produce(I1 i1, I2 i2); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImportHandler.java index ec15e037d..db1f158a9 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImportHandler.java @@ -5,9 +5,6 @@ import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.SpatialDistance; -import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; /** @@ -18,7 +15,7 @@ public class SpatialImportHandler extends DataImportHandler { public static final Logger LOGGER = LogManager.getLogger(SpatialImportHandler.class); - public SpatialImportHandler(int threads, int batchSize){ + public SpatialImportHandler(int threads, int batchSize) { super(threads, batchSize); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImporter.java index 69ca6a147..c41e9fd43 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/SpatialImporter.java @@ -8,17 +8,12 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.Location; import org.vitrivr.cineast.core.data.ReadableFloatVector; -import org.vitrivr.cineast.core.data.providers.LocationProvider; -import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.features.SpatialDistance; -import org.vitrivr.cineast.core.features.abstracts.MetadataFeatureModule; import org.vitrivr.cineast.core.importer.Importer; /** @@ -31,11 +26,11 @@ public class SpatialImporter implements Importer metadataMap; private final Map filenameToMinuteIdMap; - private final Iterator> iterator; + private final Iterator> iterator; private final Path root; - public SpatialImporter(Path path){ + public SpatialImporter(Path path) { this.root = path; try { @@ -45,36 +40,36 @@ public SpatialImporter(Path path){ metadataMap = lsc.getMetaPerMinuteId(); iterator = filenameToMinuteIdMap.entrySet().iterator(); } catch (IOException | CsvException e) { - LOGGER.error("Failed to prepare metadata readout due to {}", e,e); - throw new RuntimeException("Failed to prepare metadata readout",e); + LOGGER.error("Failed to prepare metadata readout due to {}", e, e); + throw new RuntimeException("Failed to prepare metadata readout", e); } LOGGER.info("Initialisation finished successfully. Starting import..."); } - private Optional> parseEntry(String key, String[] data){ + private Optional> parseEntry(String key, String[] data) { String lat = data[LSCUtilities.META_LAT_COL]; String lon = data[LSCUtilities.META_LON_COL]; - if(LSCUtilities.isAnyMetaBlank(lat, lon)){ - LOGGER.warn("Either lat or long for {} is blank. Ignoring.",key); + if (LSCUtilities.isAnyMetaBlank(lat, lon)) { + LOGGER.warn("Either lat or long for {} is blank. Ignoring.", key); return Optional.empty(); } boolean ignore = false; float fLat = Float.NaN, fLon = Float.NaN; - try{ - fLat = Float.parseFloat(lat); - }catch(NumberFormatException e){ - LOGGER.warn("Could not parse latitute for {}, as it was {}", key, lat,e); + try { + fLat = Float.parseFloat(lat); + } catch (NumberFormatException e) { + LOGGER.warn("Could not parse latitute for {}, as it was {}", key, lat, e); ignore = true; } - try{ + try { fLon = Float.parseFloat(lon); - }catch(NumberFormatException e){ + } catch (NumberFormatException e) { LOGGER.warn("Could not parse longitutde for {}, as it was {}", key, lon, e); ignore = true; } - if(ignore){ + if (ignore) { return Optional.empty(); } @@ -92,10 +87,10 @@ private Optional> parseEntry(String key, Stri @Override public Map readNext() { - while(iterator.hasNext()){ + while (iterator.hasNext()) { Entry next = iterator.next(); Optional> parsed = parseEntry(next.getKey(), metadataMap.get(next.getValue())); - if(parsed.isPresent()){ + if (parsed.isPresent()) { return parsed.get(); } } @@ -103,7 +98,6 @@ public Map readNext() { } - @Override public Map convert(Map data) { return data; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImportHandler.java index 373440373..8d21037f4 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImportHandler.java @@ -9,8 +9,7 @@ import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; /** - * Handler of temporal imports. - * Basically uses file (i.e. segment id) information to populate segmentstart and segmentend fields + * Handler of temporal imports. Basically uses file (i.e. segment id) information to populate segmentstart and segmentend fields */ public class TemporalImportHandler extends DataImportHandler { @@ -28,10 +27,10 @@ public TemporalImportHandler(int threads, int batchsize) { @Override - public void doImport(final Path path){ - try{ + public void doImport(final Path path) { + try { LSCUtilities.create(path); - }catch (IOException | CsvException e){ + } catch (IOException | CsvException e) { LOGGER.fatal("Error during initialisation:", e); LOGGER.fatal("Stopping immediately."); return; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImporter.java index 613ab95a6..07de1ab6a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/TemporalImporter.java @@ -22,32 +22,32 @@ public class TemporalImporter implements Importer metadataMap; private final Map filenameToMinuteIdMap; - private final Iterator> iterator; + private final Iterator> iterator; private final Path root; - public TemporalImporter(final Path path){ + public TemporalImporter(final Path path) { this.root = path; - try{ + try { final LSCUtilities lsc = LSCUtilities.create(path); lsc.initMetadata(); filenameToMinuteIdMap = lsc.getFilenameToMinuteIdLookUp(); metadataMap = lsc.getMetaPerMinuteId(); iterator = filenameToMinuteIdMap.entrySet().iterator(); - }catch (IOException | CsvException e) { - LOGGER.error("Failed to prepare metadata readout due to {}", e,e); - throw new RuntimeException("Failed to prepare metadata readout",e); + } catch (IOException | CsvException e) { + LOGGER.error("Failed to prepare metadata readout due to {}", e, e); + throw new RuntimeException("Failed to prepare metadata readout", e); } LOGGER.info("Initialisation finished successfully. Starting import now"); } - private long toEpochMillis(String minuteId){ + private long toEpochMillis(String minuteId) { final LocalDateTime date = LSCUtilities.fromMinuteId(minuteId); return date.toInstant(ZoneOffset.UTC).toEpochMilli(); } - private Optional> parseEntry(String key, String[] data){ + private Optional> parseEntry(String key, String[] data) { final long ms = toEpochMillis(filenameToMinuteIdMap.get(key)); final PrimitiveTypeProvider msProvider = PrimitiveTypeProvider.fromObject(ms); final HashMap map = new HashMap<>(); @@ -55,7 +55,7 @@ private Optional> parseEntry(String key, Stri map.put("id", PrimitiveTypeProvider.fromObject(LSCUtilities.pathToSegmentId(key))); // temporal info map.put("segmentstart", msProvider); - map.put("segmentend", PrimitiveTypeProvider.fromObject(ms+1)); // Segment ends one millis later + map.put("segmentend", PrimitiveTypeProvider.fromObject(ms + 1)); // Segment ends one millis later map.put("startabs", msProvider); return Optional.of(map); @@ -64,10 +64,10 @@ private Optional> parseEntry(String key, Stri @Override public Map readNext() { - while(iterator.hasNext()){ + while (iterator.hasNext()) { Entry next = iterator.next(); Optional> parsed = parseEntry(next.getKey(), metadataMap.get(next.getValue())); - if(parsed.isPresent()){ + if (parsed.isPresent()) { return parsed.get(); } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImportHandler.java index 7e6211f4a..fb2ec9730 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImportHandler.java @@ -1,31 +1,30 @@ package org.vitrivr.cineast.standalone.importer.lsc2020; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.features.SegmentTags; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.nio.file.Path; - public class VisualConceptTagImportHandler extends DataImportHandler { - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); - /** - * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. - * - * @param threads Number of threads to use for data import. - * @param batchsize Size of data batches that are sent to the persistence layer. - */ - public VisualConceptTagImportHandler(int threads, int batchsize) { - super(threads, batchsize); - } + /** + * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. + * + * @param threads Number of threads to use for data import. + * @param batchsize Size of data batches that are sent to the persistence layer. + */ + public VisualConceptTagImportHandler(int threads, int batchsize) { + super(threads, batchsize); + } - @Override - public void doImport(Path path) { - LOGGER.info("Starting visual concept import for tags in {}", path); - this.futures.add(this.service.submit(new DataImportRunner(new VisualConceptTagImporter(path, true), TagReader.TAG_ENTITY_NAME, "lsc-uniqueTags"))); - this.futures.add(this.service.submit(new DataImportRunner(new VisualConceptTagImporter(path), SegmentTags.SEGMENT_TAGS_TABLE_NAME, "lsc-visualConceptsTags"))); - } + @Override + public void doImport(Path path) { + LOGGER.info("Starting visual concept import for tags in {}", path); + this.futures.add(this.service.submit(new DataImportRunner(new VisualConceptTagImporter(path, true), TagReader.TAG_ENTITY_NAME, "lsc-uniqueTags"))); + this.futures.add(this.service.submit(new DataImportRunner(new VisualConceptTagImporter(path), SegmentTags.SEGMENT_TAGS_TABLE_NAME, "lsc-visualConceptsTags"))); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImporter.java index 77504226e..966d7eba0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/VisualConceptTagImporter.java @@ -2,192 +2,197 @@ import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.importer.Importer; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; - public class VisualConceptTagImporter implements Importer { - public static final Logger LOGGER = LogManager.getLogger(); + public static final Logger LOGGER = LogManager.getLogger(); - public static final String BBOX_IGNORE = "bbox"; - private Iterator content; - private Set uniqueTags; + public static final String BBOX_IGNORE = "bbox"; + private Iterator content; + private Set uniqueTags; - private static boolean ignoreColumn(String colName) { - return colName.contains(BBOX_IGNORE) || colName.contains("score") || colName.equals(LSCUtilities.KEY_MINUTEID) || colName.contains("path"); - } - - private static String scoreForCategory(String cat) { - return cat + "_score"; - } + private static boolean ignoreColumn(String colName) { + return colName.contains(BBOX_IGNORE) || colName.contains("score") || colName.equals(LSCUtilities.KEY_MINUTEID) || colName.contains("path"); + } - private static String scoreForConcept(String concept) { - return concept.replace("class", "score"); - } + private static String scoreForCategory(String cat) { + return cat + "_score"; + } - private final Path root; + private static String scoreForConcept(String concept) { + return concept.replace("class", "score"); + } - private String[] headers; + private final Path root; - private final boolean tagReaderOnly; + private String[] headers; - private Iterator tagIterator; + private final boolean tagReaderOnly; - public VisualConceptTagImporter(Path root) { - this(root, false); - } + private Iterator tagIterator; - public VisualConceptTagImporter(Path root, boolean tagReaderOnly) { - this.root = root; - this.tagReaderOnly = tagReaderOnly; - LOGGER.info("Initializing LSC Visual Concept Importer "+(tagReaderOnly ? "Tag Lookup" : "Tag Import")); - try { - readFile(); - if(tagReaderOnly){ - readTags(); - }else{ - readFileTagsPerSegment(); - } - LOGGER.info("Finished initialization."); - } catch (CsvException | IOException e) { - LOGGER.fatal("Error in reading file", e); - LOGGER.throwing(new RuntimeException("Could not initialise importer due to exception in startup", e)); - } - } + public VisualConceptTagImporter(Path root) { + this(root, false); + } - private void readFile() throws IOException, CsvException { - LOGGER.info("Reading concepts file..."); - long start = System.currentTimeMillis(); - Path file = root.resolve(LSCUtilities.CONCEPTS_FILE_NAME); - CSVReader csvReader = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); - content = csvReader.iterator(); - /* get headers */ - headers = content.next(); - LOGGER.info("Finished reading in " + (System.currentTimeMillis() - start) + "ms"); + public VisualConceptTagImporter(Path root, boolean tagReaderOnly) { + this.root = root; + this.tagReaderOnly = tagReaderOnly; + LOGGER.info("Initializing LSC Visual Concept Importer " + (tagReaderOnly ? "Tag Lookup" : "Tag Import")); + try { + readFile(); + if (tagReaderOnly) { + readTags(); + } else { + readFileTagsPerSegment(); + } + LOGGER.info("Finished initialization."); + } catch (CsvException | IOException e) { + LOGGER.fatal("Error in reading file", e); + LOGGER.throwing(new RuntimeException("Could not initialise importer due to exception in startup", e)); } - - private void readFileTagsPerSegment() { - LOGGER.info("Parsing tags per segment"); - long start = System.currentTimeMillis(); - Map> map = new HashMap<>(); - while(content.hasNext()){ - String[] line = content.next(); - String imgPath = line[LSCUtilities.CONCEPTS_IMAGEPATH_COL]; - String id = LSCUtilities.pathToSegmentId(LSCUtilities.cleanImagePath(imgPath)); - for (int i = 3; i < line.length; i++) { - if (ignoreContent(line[i])) { - continue; - } - String colName = headers[i]; - if (!ignoreColumn(colName)) { - if (map.containsKey(id)) { - map.get(id).add(line[i]); - } else { - map.put(id, new ArrayList<>()); - } - } - } + } + + private void readFile() throws IOException, CsvException { + LOGGER.info("Reading concepts file..."); + long start = System.currentTimeMillis(); + Path file = root.resolve(LSCUtilities.CONCEPTS_FILE_NAME); + CSVReader csvReader = new CSVReader(Files.newBufferedReader(file, StandardCharsets.UTF_8)); + content = csvReader.iterator(); + /* get headers */ + headers = content.next(); + LOGGER.info("Finished reading in " + (System.currentTimeMillis() - start) + "ms"); + } + + private void readFileTagsPerSegment() { + LOGGER.info("Parsing tags per segment"); + long start = System.currentTimeMillis(); + Map> map = new HashMap<>(); + while (content.hasNext()) { + String[] line = content.next(); + String imgPath = line[LSCUtilities.CONCEPTS_IMAGEPATH_COL]; + String id = LSCUtilities.pathToSegmentId(LSCUtilities.cleanImagePath(imgPath)); + for (int i = 3; i < line.length; i++) { + if (ignoreContent(line[i])) { + continue; } - mapIterator = map.entrySet().iterator(); - LOGGER.info("Finished parsing tags per segment in " + (System.currentTimeMillis() - start) + "ms"); - } - - private static boolean ignoreContent(String tag){ - return tag == null || tag.isEmpty() || tag.equals("NULL"); - } - - private void readTags() { - LOGGER.info("Parsing tags..."); - long start = System.currentTimeMillis(); - uniqueTags = new HashSet<>(); - - while(content.hasNext()){ - String[] l = content.next(); - for(int i =3; i>> mapIterator; - private volatile String currentId; - private volatile Iterator currentIterator; - - - private String[] readNextPerSegment() { - do { - if (currentId == null && mapIterator.hasNext()) { - Map.Entry> entry = mapIterator.next(); - currentId = entry.getKey(); - currentIterator = entry.getValue().iterator(); - } - if (currentIterator.hasNext()) { - return new String[]{currentId, currentIterator.next()}; - } else { - currentIterator = null; - currentId = null; - } - } while (currentIterator == null && mapIterator.hasNext()); - return null; - } - - private String[] readNextTag() { - if (tagIterator.hasNext()) { - String tag = tagIterator.next(); - return new String[]{tag, tag, ""}; // id, name, desc - } else { - return null; + String colName = headers[i]; + if (!ignoreColumn(colName)) { + if (map.containsKey(id)) { + map.get(id).add(line[i]); + } else { + map.put(id, new ArrayList<>()); + } } + } } - - @Override - public String[] readNext() { - if (tagReaderOnly) { - return readNextTag(); + mapIterator = map.entrySet().iterator(); + LOGGER.info("Finished parsing tags per segment in " + (System.currentTimeMillis() - start) + "ms"); + } + + private static boolean ignoreContent(String tag) { + return tag == null || tag.isEmpty() || tag.equals("NULL"); + } + + private void readTags() { + LOGGER.info("Parsing tags..."); + long start = System.currentTimeMillis(); + uniqueTags = new HashSet<>(); + + while (content.hasNext()) { + String[] l = content.next(); + for (int i = 3; i < l.length; i++) { + if (ignoreColumn(headers[i])) { + continue; } else { - return readNextPerSegment(); + if (!ignoreContent(l[i])) { + uniqueTags.add(l[i]); + } } + } } - - private Map convertPerSegment(String[] data) { - Map map = new HashMap<>(); - map.put("id", PrimitiveTypeProvider.fromObject(data[0])); - map.put("tagid", PrimitiveTypeProvider.fromObject(data[1])); - map.put("score", PrimitiveTypeProvider.fromObject(1)); - return map; + tagIterator = uniqueTags.iterator(); + LOGGER.info("Finished parsing tags in " + (System.currentTimeMillis() - start) + "ms"); + } + + private volatile Iterator>> mapIterator; + private volatile String currentId; + private volatile Iterator currentIterator; + + + private String[] readNextPerSegment() { + do { + if (currentId == null && mapIterator.hasNext()) { + Map.Entry> entry = mapIterator.next(); + currentId = entry.getKey(); + currentIterator = entry.getValue().iterator(); + } + if (currentIterator.hasNext()) { + return new String[]{currentId, currentIterator.next()}; + } else { + currentIterator = null; + currentId = null; + } + } while (currentIterator == null && mapIterator.hasNext()); + return null; + } + + private String[] readNextTag() { + if (tagIterator.hasNext()) { + String tag = tagIterator.next(); + return new String[]{tag, tag, ""}; // id, name, desc + } else { + return null; } - - private Map convertAsTag(String[] data) { - Map map = new HashMap<>(); - map.put(TagReader.TAG_ID_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[0])); - map.put(TagReader.TAG_NAME_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[1])); - map.put(TagReader.TAG_DESCRIPTION_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[1])); // LSC Context: No description available. Use label instead - return map; + } + + @Override + public String[] readNext() { + if (tagReaderOnly) { + return readNextTag(); + } else { + return readNextPerSegment(); } - - @Override - public Map convert(String[] data) { - if (tagReaderOnly) { - return convertAsTag(data); - } else { - return convertPerSegment(data); - } + } + + private Map convertPerSegment(String[] data) { + Map map = new HashMap<>(); + map.put("id", PrimitiveTypeProvider.fromObject(data[0])); + map.put("tagid", PrimitiveTypeProvider.fromObject(data[1])); + map.put("score", PrimitiveTypeProvider.fromObject(1)); + return map; + } + + private Map convertAsTag(String[] data) { + Map map = new HashMap<>(); + map.put(TagReader.TAG_ID_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[0])); + map.put(TagReader.TAG_NAME_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[1])); + map.put(TagReader.TAG_DESCRIPTION_COLUMNNAME, PrimitiveTypeProvider.fromObject(data[1])); // LSC Context: No description available. Use label instead + return map; + } + + @Override + public Map convert(String[] data) { + if (tagReaderOnly) { + return convertAsTag(data); + } else { + return convertPerSegment(data); } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptImportHandler.java index 57869b4b9..ce0e5420e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptImportHandler.java @@ -1,15 +1,14 @@ package org.vitrivr.cineast.standalone.importer.vbs2019; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.features.AudioTranscriptionSearch; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - public class AudioTranscriptImportHandler extends DataImportHandler { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptionImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptionImporter.java index 11eb72f15..64e22a7eb 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptionImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/AudioTranscriptionImporter.java @@ -5,13 +5,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.Pair; -import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.importer.Importer; - import java.io.IOException; import java.nio.file.Path; import java.util.HashMap; @@ -19,6 +12,12 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.importer.Importer; public class AudioTranscriptionImporter implements Importer> { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImportHandler.java index 09a7e0dfc..0048a3975 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImportHandler.java @@ -1,15 +1,14 @@ package org.vitrivr.cineast.standalone.importer.vbs2019; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.features.DescriptionTextSearch; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - public class CaptionTextImportHandler extends DataImportHandler { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImporter.java index a68ea90f6..cccd2b642 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/CaptionTextImporter.java @@ -5,6 +5,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.Optional; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -12,11 +20,6 @@ import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.importer.Importer; -import java.io.IOException; -import java.nio.file.Path; -import java.util.*; -import java.util.Map.Entry; - public class CaptionTextImporter implements Importer> { private final Iterator> videos; @@ -41,7 +44,7 @@ public CaptionTextImporter(Path input) throws IOException { } private synchronized Optional> nextPair() { - while(currentSegments == null || !currentSegments.hasNext()){ + while (currentSegments == null || !currentSegments.hasNext()) { Entry next = videos.next(); currentVideoID = next.getKey(); currentSegments = next.getValue().fields(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/GoogleVisionImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/GoogleVisionImporter.java index fd4d6af7a..20491a0f7 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/GoogleVisionImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/GoogleVisionImporter.java @@ -26,245 +26,223 @@ /** * Import code for the google vision export of the V3C1 Dataset. - * - * Format is as follows: - * [ // whole dataset - * { // video - * "segmentid": { - * "web": [ //list of concepts - * "id": "/m/...", - * "decription" (not a typo): "Human-readable description, capitalized" - * "score:" score (can be above 1) - * ], - * "ocr": [ - * "id": "", - * "decription": "text", - * "score": 0.0 - * ], - * "labels": [ - * "id": "/m/...", - * "decription": "human-readable description, non-capitalized" - * "score": score between 0 and 1 - * ] - * }, - * // list of segments - * }, - * // list of videos + *

    + * Format is as follows: [ // whole dataset { // video "segmentid": { "web": [ //list of concepts "id": "/m/...", "decription" (not a typo): "Human-readable description, capitalized" "score:" score (can be above 1) ], "ocr": [ "id": "", "decription": "text", "score": 0.0 ], "labels": [ "id": "/m/...", "decription": "human-readable description, non-capitalized" "score": score between 0 and 1 ] }, // list of segments }, // list of videos */ public class GoogleVisionImporter implements Importer { - private JsonParser parser; - private ObjectMapper mapper; - private final String fileName; - private final Path input; - private Iterator> _segments; - private Iterator> _categories; - private static final Logger LOGGER = LogManager.getLogger(); - private String _completeID; - private final GoogleVisionCategory targetCategory; - private Iterator _categoryValues; - private GoogleVisionCategory _category; - private final boolean importTagsFt; - private boolean initialized = false; + private JsonParser parser; + private ObjectMapper mapper; + private final String fileName; + private final Path input; + private Iterator> _segments; + private Iterator> _categories; + private static final Logger LOGGER = LogManager.getLogger(); + private String _completeID; + private final GoogleVisionCategory targetCategory; + private Iterator _categoryValues; + private GoogleVisionCategory _category; + private final boolean importTagsFt; + private boolean initialized = false; - /** - * @param importTagsFt whether tags should be imported into {@link TagsFtSearch} or {@link GoogleVisionCategory#tableName} - * @param targetCategory only tuples of this kind are imported - */ - public GoogleVisionImporter(Path input, GoogleVisionCategory targetCategory, boolean importTagsFt) throws IOException { - this.input = input; - this.fileName = input.getFileName().toString(); - this.importTagsFt = importTagsFt; - this.targetCategory = targetCategory; - } + /** + * @param importTagsFt whether tags should be imported into {@link TagsFtSearch} or {@link GoogleVisionCategory#tableName} + * @param targetCategory only tuples of this kind are imported + */ + public GoogleVisionImporter(Path input, GoogleVisionCategory targetCategory, boolean importTagsFt) throws IOException { + this.input = input; + this.fileName = input.getFileName().toString(); + this.importTagsFt = importTagsFt; + this.targetCategory = targetCategory; + } - /** - * Generate a {@link GoogleVisionTuple} from the current state - */ - private Optional generateTuple() { - JsonNode next = _categoryValues.next(); - try { - return Optional.of(GoogleVisionTuple.of(targetCategory, next, _completeID)); - } catch (UnsupportedOperationException e) { - LOGGER.trace("Cannot generate tuple for category {} and tuple {}", targetCategory, next); - return Optional.empty(); - } + /** + * Generate a {@link GoogleVisionTuple} from the current state + */ + private Optional generateTuple() { + JsonNode next = _categoryValues.next(); + try { + return Optional.of(GoogleVisionTuple.of(targetCategory, next, _completeID)); + } catch (UnsupportedOperationException e) { + LOGGER.trace("Cannot generate tuple for category {} and tuple {}", targetCategory, next); + return Optional.empty(); } + } - /** - * Search for the next valid {@link GoogleVisionTuple} in the current segment - */ - private Optional searchWithinSegment() { - //First, check if there's still a value left in the current array - if (_category == targetCategory && _categoryValues.hasNext()) { - return generateTuple(); - } + /** + * Search for the next valid {@link GoogleVisionTuple} in the current segment + */ + private Optional searchWithinSegment() { + //First, check if there's still a value left in the current array + if (_category == targetCategory && _categoryValues.hasNext()) { + return generateTuple(); + } - //if not, check if we can get values for the target category in the given segment - while (_categories != null && _category != targetCategory && _categories.hasNext()) { - Entry nextCategory = _categories.next(); - _category = GoogleVisionCategory.valueOf(nextCategory.getKey().toUpperCase()); - _categoryValues = nextCategory.getValue().iterator(); - } + //if not, check if we can get values for the target category in the given segment + while (_categories != null && _category != targetCategory && _categories.hasNext()) { + Entry nextCategory = _categories.next(); + _category = GoogleVisionCategory.valueOf(nextCategory.getKey().toUpperCase()); + _categoryValues = nextCategory.getValue().iterator(); + } - //If we succeeded in the previous loop, we should be at the target category and still have a value left to hand out. - if (_category == targetCategory && _categoryValues.hasNext()) { - return generateTuple(); - } - //else we have nothing in this category - return Optional.empty(); + //If we succeeded in the previous loop, we should be at the target category and still have a value left to hand out. + if (_category == targetCategory && _categoryValues.hasNext()) { + return generateTuple(); } + //else we have nothing in this category + return Optional.empty(); + } - private Optional searchWithinMovie() { - do { - Optional tuple = searchWithinSegment(); - if (tuple.isPresent()) { - return tuple; - } - //we need to move on to the next segment - if (!_segments.hasNext()) { - return Optional.empty(); - } - Entry nextSegment = _segments.next(); - _completeID = nextSegment.getKey(); - _categories = nextSegment.getValue().fields(); - //Initialize category values - Entry nextCategory = _categories.next(); - _category = GoogleVisionCategory.valueOf(nextCategory.getKey().toUpperCase()); - _categoryValues = nextCategory.getValue().iterator(); - } while (_segments.hasNext()); + private Optional searchWithinMovie() { + do { + Optional tuple = searchWithinSegment(); + if (tuple.isPresent()) { + return tuple; + } + //we need to move on to the next segment + if (!_segments.hasNext()) { return Optional.empty(); - } + } + Entry nextSegment = _segments.next(); + _completeID = nextSegment.getKey(); + _categories = nextSegment.getValue().fields(); + //Initialize category values + Entry nextCategory = _categories.next(); + _category = GoogleVisionCategory.valueOf(nextCategory.getKey().toUpperCase()); + _categoryValues = nextCategory.getValue().iterator(); + } while (_segments.hasNext()); + return Optional.empty(); + } - /** - * Gets the next {@link GoogleVisionTuple} to Import. - */ - private synchronized Optional nextTuple() { - //Check first if there are tuples left within the current movie / aggregation of segments - Optional tuple = searchWithinMovie(); - if (tuple.isPresent()) { - return tuple; - } -//if the current movie has no tuples left to import, we need to go to the next movie until we find a tuple - do { - //we need to go to the next movie - try { - if (parser.nextToken() == JsonToken.START_OBJECT) { - ObjectNode nextMovie = mapper.readTree(parser); - if (nextMovie == null) { - LOGGER.info("File for category {} is done", targetCategory); - return Optional.empty(); - } - _segments = nextMovie.fields(); - tuple = searchWithinMovie(); - if (tuple.isPresent()) { - return tuple; - } - } else { - LOGGER.info("File {} done", this.fileName); - return Optional.empty(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } while (true); + /** + * Gets the next {@link GoogleVisionTuple} to Import. + */ + private synchronized Optional nextTuple() { + //Check first if there are tuples left within the current movie / aggregation of segments + Optional tuple = searchWithinMovie(); + if (tuple.isPresent()) { + return tuple; } - - private synchronized void init() throws IOException { - if (initialized) { - LOGGER.warn("Importer for path {} was already initalized", input); - return; - } - initialized = true; - LOGGER.info("Starting Importer for path {}, category {} and importTags {}", input, targetCategory, importTagsFt); - mapper = new ObjectMapper(); - parser = mapper.getFactory().createParser(input.toFile()); - if (parser.nextToken() == JsonToken.START_ARRAY) { - if (parser.nextToken() == JsonToken.START_OBJECT) { - ObjectNode node = mapper.readTree(parser); - _segments = node.fields(); - } - if (_segments == null) { - throw new IOException("Empty file"); - } +//if the current movie has no tuples left to import, we need to go to the next movie until we find a tuple + do { + //we need to go to the next movie + try { + if (parser.nextToken() == JsonToken.START_OBJECT) { + ObjectNode nextMovie = mapper.readTree(parser); + if (nextMovie == null) { + LOGGER.info("File for category {} is done", targetCategory); + return Optional.empty(); + } + _segments = nextMovie.fields(); + tuple = searchWithinMovie(); + if (tuple.isPresent()) { + return tuple; + } } else { - throw new IOException("Empty file"); + LOGGER.info("File {} done", this.fileName); + return Optional.empty(); } + } catch (IOException e) { + throw new RuntimeException(e); + } + } while (true); + } + + private synchronized void init() throws IOException { + if (initialized) { + LOGGER.warn("Importer for path {} was already initalized", input); + return; + } + initialized = true; + LOGGER.info("Starting Importer for path {}, category {} and importTags {}", input, targetCategory, importTagsFt); + mapper = new ObjectMapper(); + parser = mapper.getFactory().createParser(input.toFile()); + if (parser.nextToken() == JsonToken.START_ARRAY) { + if (parser.nextToken() == JsonToken.START_OBJECT) { + ObjectNode node = mapper.readTree(parser); + _segments = node.fields(); + } + if (_segments == null) { + throw new IOException("Empty file"); + } + } else { + throw new IOException("Empty file"); } + } - /** - * @return Pair mapping a segmentID to a List of Descriptions - */ - @Override - public GoogleVisionTuple readNext() { - try { - if (!initialized) { - init(); - } - Optional node = nextTuple(); - if (!node.isPresent()) { - return null; - } - return node.get(); - } catch (NoSuchElementException | IOException e) { - e.printStackTrace(); - return null; - } + /** + * @return Pair mapping a segmentID to a List of Descriptions + */ + @Override + public GoogleVisionTuple readNext() { + try { + if (!initialized) { + init(); + } + Optional node = nextTuple(); + if (!node.isPresent()) { + return null; + } + return node.get(); + } catch (NoSuchElementException | IOException e) { + e.printStackTrace(); + return null; } + } - /** - * Converts the given {@link GoogleVisionTuple} to a representation appropriate to the given feature. - * - * @param data the tuple to be converted to a tuple in the feature table - * @return a map where the key corresponds to the column-name and the value to the value to be inserted in that column for the given tuple - */ - @Override - public Map convert(GoogleVisionTuple data) { - final HashMap map = new HashMap<>(2); - Optional tag = Optional.empty(); - PrimitiveTypeProvider id = PrimitiveTypeProvider.fromObject(data.completeID); - switch (data.category) { - case PARTIALLY_MATCHING_IMAGES: - throw new UnsupportedOperationException(); - case WEB: - if (importTagsFt) { - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.web.get().description)); - } else { - map.put("id", id); - map.put("tagid", PrimitiveTypeProvider.fromObject(data.web.get().labelId)); - map.put("score", PrimitiveTypeProvider.fromObject(Math.min(1, data.web.get().score))); - try { - tag = Optional.of(new CompleteTag(data.web.get().labelId, data.web.get().description, data.web.get().description)); - } catch (IllegalArgumentException e) { - LOGGER.trace("Error while initalizing tag {}", e.getMessage()); - } - } - break; - case PAGES_MATCHING_IMAGES: - throw new UnsupportedOperationException(); - case FULLY_MATCHING_IMAGES: - throw new UnsupportedOperationException(); - case LABELS: - if (importTagsFt) { - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.label.get().description)); - } else { - map.put("id", id); - map.put("tagid", PrimitiveTypeProvider.fromObject(data.label.get().labelId)); - map.put("score", PrimitiveTypeProvider.fromObject(Math.min(1, data.label.get().score))); - tag = Optional.of(new CompleteTag(data.label.get().labelId, data.label.get().description, data.label.get().description)); - } - break; - case OCR: - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); - map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.ocr.get().description)); - // Score is ignored because it's never used in search and 0 anyways - //map.put("score", PrimitiveTypeProvider.fromObject(data.ocr.get().score)); - break; - default: - throw new UnsupportedOperationException(); + /** + * Converts the given {@link GoogleVisionTuple} to a representation appropriate to the given feature. + * + * @param data the tuple to be converted to a tuple in the feature table + * @return a map where the key corresponds to the column-name and the value to the value to be inserted in that column for the given tuple + */ + @Override + public Map convert(GoogleVisionTuple data) { + final HashMap map = new HashMap<>(2); + Optional tag = Optional.empty(); + PrimitiveTypeProvider id = PrimitiveTypeProvider.fromObject(data.completeID); + switch (data.category) { + case PARTIALLY_MATCHING_IMAGES: + throw new UnsupportedOperationException(); + case WEB: + if (importTagsFt) { + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.web.get().description)); + } else { + map.put("id", id); + map.put("tagid", PrimitiveTypeProvider.fromObject(data.web.get().labelId)); + map.put("score", PrimitiveTypeProvider.fromObject(Math.min(1, data.web.get().score))); + try { + tag = Optional.of(new CompleteTag(data.web.get().labelId, data.web.get().description, data.web.get().description)); + } catch (IllegalArgumentException e) { + LOGGER.trace("Error while initalizing tag {}", e.getMessage()); + } + } + break; + case PAGES_MATCHING_IMAGES: + throw new UnsupportedOperationException(); + case FULLY_MATCHING_IMAGES: + throw new UnsupportedOperationException(); + case LABELS: + if (importTagsFt) { + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.label.get().description)); + } else { + map.put("id", id); + map.put("tagid", PrimitiveTypeProvider.fromObject(data.label.get().labelId)); + map.put("score", PrimitiveTypeProvider.fromObject(Math.min(1, data.label.get().score))); + tag = Optional.of(new CompleteTag(data.label.get().labelId, data.label.get().description, data.label.get().description)); } - return map; + break; + case OCR: + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[0], id); + map.put(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject(data.ocr.get().description)); + // Score is ignored because it's never used in search and 0 anyways + //map.put("score", PrimitiveTypeProvider.fromObject(data.ocr.get().score)); + break; + default: + throw new UnsupportedOperationException(); } + return map; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeatureImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeatureImporter.java index 061799b26..7f62a4cef 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeatureImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeatureImporter.java @@ -8,7 +8,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; - import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; @@ -16,12 +15,10 @@ import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.importer.Importer; -import javax.swing.text.html.Option; - /** * Purpose-built for output generated by https://github.com/vitrivr/mobilenet-feature-extraction */ -public class MLTFeatureImporter implements Importer> { +public class MLTFeatureImporter implements Importer> { private static final Logger LOGGER = LogManager.getLogger(); private final BufferedReader reader; @@ -33,7 +30,7 @@ public MLTFeatureImporter(Path input) throws IOException { private synchronized Optional> nextPair() { try { String line = reader.readLine(); - if(line==null){ + if (line == null) { LOGGER.info("Reached EoF"); return Optional.empty(); } @@ -41,7 +38,7 @@ private synchronized Optional> nextPair() { String id = split[0]; float[] feature = new float[512]; for (int i = 1; i < split.length; i++) { - feature[i-1] = Float.parseFloat(split[i]); + feature[i - 1] = Float.parseFloat(split[i]); } return Optional.of(ImmutablePair.of(id, feature)); } catch (IOException e) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeaturesImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeaturesImportHandler.java index 9a12e0efa..0af396a8c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeaturesImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/MLTFeaturesImportHandler.java @@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.DescriptionTextSearch; import org.vitrivr.cineast.core.features.ObjectInstances; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.config.Config; @@ -18,7 +17,7 @@ public class MLTFeaturesImportHandler extends DataImportHandler { public MLTFeaturesImportHandler(int threads, int batchsize, boolean clean) { super(threads, batchsize); - if(clean){ + if (clean) { final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); LOGGER.info("Dropping table {}", ObjectInstances.TABLE_NAME); ec.dropEntity(ObjectInstances.TABLE_NAME); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImportHandler.java index 817b22be8..e51f40023 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImportHandler.java @@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; -import org.vitrivr.cineast.core.features.AudioTranscriptionSearch; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImporter.java index 483687bb2..80f6c2f1e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/ObjectMetadataImporter.java @@ -24,107 +24,107 @@ public class ObjectMetadataImporter implements Importer> { - private static final Logger LOGGER = LogManager.getLogger(); - private final ObjectMapper mapper; - private final JsonParser parser; - private String _id; - private Iterator> _meta; - private Iterator _metaArray; - private String _key; + private static final Logger LOGGER = LogManager.getLogger(); + private final ObjectMapper mapper; + private final JsonParser parser; + private String _id; + private Iterator> _meta; + private Iterator _metaArray; + private String _key; - public ObjectMetadataImporter(Path input) throws IOException { - mapper = new ObjectMapper(); - parser = mapper.getFactory().createParser(input.toFile()); - if (parser.nextToken() != JsonToken.START_ARRAY) { - throw new IOException("Invalid format"); - } - if (parser.nextToken() != JsonToken.START_OBJECT) { - throw new IOException("Invalid format"); - } - nextObjectMetadata(); + public ObjectMetadataImporter(Path input) throws IOException { + mapper = new ObjectMapper(); + parser = mapper.getFactory().createParser(input.toFile()); + if (parser.nextToken() != JsonToken.START_ARRAY) { + throw new IOException("Invalid format"); } - - private Optional>> nextObjectMetadata() throws IOException { - if (parser.nextToken() != JsonToken.START_OBJECT) { - LOGGER.info("File done"); - return Optional.empty(); - } - ObjectNode node = mapper.readTree(parser); - if (node == null) { - LOGGER.info("File is done"); - return Optional.empty(); - } - _meta = node.fields(); - if (_meta == null) { - throw new IOException("Empty file"); - } - _id = "v_" + node.get("v3cId").textValue(); - return Optional.of(_meta); + if (parser.nextToken() != JsonToken.START_OBJECT) { + throw new IOException("Invalid format"); } + nextObjectMetadata(); + } - private synchronized Optional> nextPair() throws IOException { - while (_meta == null || !_meta.hasNext()) { - //isempty() only after java 11 - if (!nextObjectMetadata().isPresent()) { - return Optional.empty(); - } - } - if (_metaArray != null && _metaArray.hasNext()) { - //return active meta array element - } - Entry next = _meta.next(); - JsonNode jsonVal = next.getValue(); - String key = next.getKey(); - PrimitiveTypeProvider primitiveVal = null; - if (next.getValue().isArray()) { - _metaArray = next.getValue().iterator(); - _key = next.getKey(); - if (_metaArray.hasNext()) { - jsonVal = _metaArray.next(); - } else { - return nextPair(); - } - } - if (jsonVal.isFloatingPointNumber()) { - primitiveVal = new DoubleTypeProvider(jsonVal.asDouble()); - } - if (jsonVal.isNumber()) { - primitiveVal = new LongTypeProvider(jsonVal.asLong()); - } - if (jsonVal.isTextual()) { - primitiveVal = new StringTypeProvider(jsonVal.asText()); - } - if (primitiveVal == null) { - LOGGER.warn("Unknown type {}", jsonVal.getNodeType()); - } - - final HashMap map = new HashMap<>(2); - map.put(MediaObjectMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(_id)); - map.put(MediaObjectMetadataDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject("vimeo")); - map.put(MediaObjectMetadataDescriptor.FIELDNAMES[2], PrimitiveTypeProvider.fromObject(key)); - map.put(MediaObjectMetadataDescriptor.FIELDNAMES[3], primitiveVal); - return Optional.of(map); + private Optional>> nextObjectMetadata() throws IOException { + if (parser.nextToken() != JsonToken.START_OBJECT) { + LOGGER.info("File done"); + return Optional.empty(); + } + ObjectNode node = mapper.readTree(parser); + if (node == null) { + LOGGER.info("File is done"); + return Optional.empty(); } + _meta = node.fields(); + if (_meta == null) { + throw new IOException("Empty file"); + } + _id = "v_" + node.get("v3cId").textValue(); + return Optional.of(_meta); + } - /** - * @return Pair mapping a segmentID to a List of Descriptions - */ - @Override - public Map readNext() { - try { - Optional> node = nextPair(); - //isEmpty() only since java 11 - if (!node.isPresent()) { - return null; - } - return node.get(); - } catch (NoSuchElementException | IOException e) { - return null; - } + private synchronized Optional> nextPair() throws IOException { + while (_meta == null || !_meta.hasNext()) { + //isempty() only after java 11 + if (!nextObjectMetadata().isPresent()) { + return Optional.empty(); + } + } + if (_metaArray != null && _metaArray.hasNext()) { + //return active meta array element + } + Entry next = _meta.next(); + JsonNode jsonVal = next.getValue(); + String key = next.getKey(); + PrimitiveTypeProvider primitiveVal = null; + if (next.getValue().isArray()) { + _metaArray = next.getValue().iterator(); + _key = next.getKey(); + if (_metaArray.hasNext()) { + jsonVal = _metaArray.next(); + } else { + return nextPair(); + } } + if (jsonVal.isFloatingPointNumber()) { + primitiveVal = new DoubleTypeProvider(jsonVal.asDouble()); + } + if (jsonVal.isNumber()) { + primitiveVal = new LongTypeProvider(jsonVal.asLong()); + } + if (jsonVal.isTextual()) { + primitiveVal = new StringTypeProvider(jsonVal.asText()); + } + if (primitiveVal == null) { + LOGGER.warn("Unknown type {}", jsonVal.getNodeType()); + } + + final HashMap map = new HashMap<>(2); + map.put(MediaObjectMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject(_id)); + map.put(MediaObjectMetadataDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject("vimeo")); + map.put(MediaObjectMetadataDescriptor.FIELDNAMES[2], PrimitiveTypeProvider.fromObject(key)); + map.put(MediaObjectMetadataDescriptor.FIELDNAMES[3], primitiveVal); + return Optional.of(map); + } - @Override - public Map convert(Map data) { - return data; + /** + * @return Pair mapping a segmentID to a List of Descriptions + */ + @Override + public Map readNext() { + try { + Optional> node = nextPair(); + //isEmpty() only since java 11 + if (!node.isPresent()) { + return null; + } + return node.get(); + } catch (NoSuchElementException | IOException e) { + return null; } + } + + @Override + public Map convert(Map data) { + return data; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/gvision/GoogleVisionCategory.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/gvision/GoogleVisionCategory.java index 971d668d5..cf74e2f09 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/gvision/GoogleVisionCategory.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/gvision/GoogleVisionCategory.java @@ -6,7 +6,7 @@ public enum GoogleVisionCategory { public final String tableName; - GoogleVisionCategory(String tableName){ + GoogleVisionCategory(String tableName) { this.tableName = tableName; } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImporter.java index d76ab2e5a..face0bd88 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImporter.java @@ -14,7 +14,7 @@ /** * Imports the colorlabels as given by the https://github.com/klschoef/V3C1Analysis repo. File is expected to be in format: - * + *

    * every line contains the segmentid formatted xxxxx_yyyy, where before the _ is 0-padded to length 5 the movie id and behind the _ the non-0 padded segmentID */ public class ColorlabelImporter implements Importer> { @@ -37,7 +37,7 @@ private synchronized Optional> nextPair() { } String id = lineIterator.next(); Map _return = new HashMap<>(); - _return.put(MediaSegmentMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject("v_"+id)); + _return.put(MediaSegmentMetadataDescriptor.FIELDNAMES[0], PrimitiveTypeProvider.fromObject("v_" + id)); _return.put(MediaSegmentMetadataDescriptor.FIELDNAMES[1], PrimitiveTypeProvider.fromObject("v3c1")); _return.put(MediaSegmentMetadataDescriptor.FIELDNAMES[2], PrimitiveTypeProvider.fromObject("colorlabels")); _return.put(MediaSegmentMetadataDescriptor.FIELDNAMES[3], PrimitiveTypeProvider.fromObject(this.label)); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultCSVExporter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultCSVExporter.java index d0e0eb7d8..ff8bd121c 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultCSVExporter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultCSVExporter.java @@ -1,5 +1,13 @@ package org.vitrivr.cineast.standalone.listener; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; @@ -12,11 +20,6 @@ import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.standalone.runtime.RetrievalTask; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintWriter; -import java.util.*; - public class RetrievalResultCSVExporter implements RetrievalResultListener { private static File baseFolder = new File("retrieval_results"); // TODO make configurable @@ -25,7 +28,7 @@ public class RetrievalResultCSVExporter implements RetrievalResultListener { private final MediaSegmentReader mediaSegmentReader; private final MediaObjectReader mediaObjectReader; - public RetrievalResultCSVExporter(DatabaseConfig databaseConfig){ + public RetrievalResultCSVExporter(DatabaseConfig databaseConfig) { mediaSegmentReader = new MediaSegmentReader(databaseConfig.getSelectorSupplier().get()); mediaObjectReader = new MediaObjectReader(databaseConfig.getSelectorSupplier().get()); } @@ -46,26 +49,25 @@ public void notify(List resultList, RetrievalTask task) { outFolder.mkdirs(); File out = new File(outFolder, filename); - ArrayList ids = new ArrayList<>(resultList.size()); - for(ScoreElement e : resultList){ + for (ScoreElement e : resultList) { ids.add(e.getId()); } - + Map segments = mediaSegmentReader.lookUpSegments(ids); Set objectIds = new HashSet<>(); - - for(MediaSegmentDescriptor sd : segments.values()){ + + for (MediaSegmentDescriptor sd : segments.values()) { objectIds.add(sd.getObjectId()); } - + Map objects = mediaObjectReader.lookUpObjects(objectIds); - + try (PrintWriter writer = new PrintWriter(out)) { - + //header writer.println("\"rank\", \"id\", \"score\", \"path\""); - + int rank = 1; for (ScoreElement e : resultList) { writer.print(rank++); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultListener.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultListener.java index 15e9e0ec8..8a91c083b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultListener.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/listener/RetrievalResultListener.java @@ -1,11 +1,10 @@ package org.vitrivr.cineast.standalone.listener; +import java.util.List; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.standalone.runtime.RetrievalTask; -import java.util.List; - /** * Listener which can be attached to retrieval logic to get notified on raw results of a {@link Retriever} */ @@ -13,9 +12,10 @@ public interface RetrievalResultListener { /** * gets called whenever a {@link Retriever} returns + * * @param resultList the retrieved results - * @param task the {@link RetrievalTask} containing retriever and query + * @param task the {@link RetrievalTask} containing retriever and query */ public void notify(List resultList, RetrievalTask task); - + } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java index 5d3d9192c..9391c9468 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java @@ -1,6 +1,5 @@ package org.vitrivr.cineast.standalone.monitoring; -import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import io.prometheus.client.Summary; import org.apache.logging.log4j.LogManager; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java index cd07a6b8a..d247fe1e0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusExtractionTaskMonitor.java @@ -6,8 +6,7 @@ import org.vitrivr.cineast.standalone.config.Config; /** - * So we don't clutter the ExtractionTask code. - * Singleton where you can register an extraction time of features + * So we don't clutter the ExtractionTask code. Singleton where you can register an extraction time of features */ public class PrometheusExtractionTaskMonitor extends ImportTaskMonitor { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java index 5e0aec8df..7e2dad6d4 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java @@ -13,7 +13,6 @@ /** * Singleton which starts a reporting endpoint for Prometheus - * */ public class PrometheusServer { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java index defca29b1..4ab96ccd6 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionCompleteListener.java @@ -5,18 +5,17 @@ */ public interface ExtractionCompleteListener { - /** - * This method is called after the decoder for the object has been closed. - * There might still be scheduled or ongoing ExtractionTasks for this object. - */ - default void onCompleted(ExtractionItemContainer path){ - //ignore - } + /** + * This method is called after the decoder for the object has been closed. There might still be scheduled or ongoing ExtractionTasks for this object. + */ + default void onCompleted(ExtractionItemContainer path) { + //ignore + } - /** - * This method is called when the extraction is completely finished. All resources are relinquished and no tasks are ongoing anymore. - */ - default void extractionComplete(){ - //ignore - } + /** + * This method is called when the extraction is completely finished. All resources are relinquished and no tasks are ongoing anymore. + */ + default void extractionComplete() { + //ignore + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionContainerProvider.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionContainerProvider.java index 5da2e3665..eddeab278 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionContainerProvider.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionContainerProvider.java @@ -1,15 +1,11 @@ package org.vitrivr.cineast.standalone.run; import com.google.common.collect.Lists; - import java.util.List; import java.util.Optional; /** - * The {@link ExtractionContainerProvider} provides a continuous list of {@link - * ExtractionItemContainer}. It is intended to be both used for e.g. walking a directory or during - * an extraction-session using Cineast's API - * + * The {@link ExtractionContainerProvider} provides a continuous list of {@link ExtractionItemContainer}. It is intended to be both used for e.g. walking a directory or during an extraction-session using Cineast's API */ public interface ExtractionContainerProvider { @@ -25,15 +21,12 @@ default void addPath(ExtractionItemContainer path) { } /** - * Check if this instance is still active. This does NOT mean that {@link #hasNextAvailable()} - * will return true, but simply that there might be elements yet to come. + * Check if this instance is still active. This does NOT mean that {@link #hasNextAvailable()} will return true, but simply that there might be elements yet to come. */ boolean isOpen(); /** - * Check if {@link #next()} could return an element. Since a {@link ExtractionContainerProvider} - * might be used in a multi-threaded context, it does not guarantee that by the time you will call - * {@link #next()}, the element will still be available. + * Check if {@link #next()} could return an element. Since a {@link ExtractionContainerProvider} might be used in a multi-threaded context, it does not guarantee that by the time you will call {@link #next()}, the element will still be available. */ boolean hasNextAvailable(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionDispatcher.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionDispatcher.java index a6b53df48..f5d46ed82 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionDispatcher.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionDispatcher.java @@ -1,13 +1,12 @@ package org.vitrivr.cineast.standalone.run; +import java.io.File; +import java.io.IOException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.config.IngestConfig; -import java.io.File; -import java.io.IOException; - public class ExtractionDispatcher { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java index f506a885a..67f861fb2 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java @@ -13,11 +13,7 @@ import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; /** - * An {@link ExtractionItemContainer} contains all information for ONE item which is supposed to be - * extracted. A container MUST contain a {@link #path} linking to the item to be extracted. The - * corresponding {@link #object} MUST contain the {@link MediaObjectDescriptor#getMediatype()} so an - * item can be handed out to different extractionhandlers. - * + * An {@link ExtractionItemContainer} contains all information for ONE item which is supposed to be extracted. A container MUST contain a {@link #path} linking to the item to be extracted. The corresponding {@link #object} MUST contain the {@link MediaObjectDescriptor#getMediatype()} so an item can be handed out to different extractionhandlers. */ public class ExtractionItemContainer { @@ -62,8 +58,7 @@ public String getUri() { } /** - * If a path has been specified directly, returns the path. If no path has been specified, tries - * to construct a path from the path of the underlying {@link MediaObjectDescriptor} + * If a path has been specified directly, returns the path. If no path has been specified, tries to construct a path from the path of the underlying {@link MediaObjectDescriptor} */ @JsonIgnore public Path getPathForExtraction() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemProcessor.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemProcessor.java index ae2c2552c..f16c17554 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemProcessor.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemProcessor.java @@ -2,7 +2,6 @@ /** * An {@link ExtractionItemProcessor} processes {@link ExtractionItemContainer}s. - * */ public interface ExtractionItemProcessor { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java index 2e163c284..8e774d596 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java @@ -61,7 +61,6 @@ * This class is used to extract a continuous list of {@link ExtractionItemContainer}s. *

    * Additionally, has support to extract only specific media types by providing the desired {@link MediaType} in the constructor. - * */ public class GenericExtractionItemHandler implements Runnable, ExtractionItemProcessor { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/ExtractionContainerProviderFactory.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/ExtractionContainerProviderFactory.java index fbc714f64..a152b0691 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/ExtractionContainerProviderFactory.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/ExtractionContainerProviderFactory.java @@ -1,26 +1,24 @@ package org.vitrivr.cineast.standalone.run.path; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.extraction.ExtractionContextProvider; import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; - public class ExtractionContainerProviderFactory { private static final Logger LOGGER = LogManager.getLogger(); /** - * Tries to create a {@link TreeWalkContainerIteratorProvider}. Will however create a {@link - * NoContainerProvider} if something goes wrong. + * Tries to create a {@link TreeWalkContainerIteratorProvider}. Will however create a {@link NoContainerProvider} if something goes wrong. */ public static ExtractionContainerProvider tryCreatingTreeWalkPathProvider(File jobFile, ExtractionContextProvider context) { - /* Check if context could be read and an input path was specified. */ + /* Check if context could be read and an input path was specified. */ if (context == null || !context.inputPath().isPresent()) { return new NoContainerProvider(); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/NoContainerProvider.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/NoContainerProvider.java index 05c1ca94d..863d0ad60 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/NoContainerProvider.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/NoContainerProvider.java @@ -1,15 +1,13 @@ package org.vitrivr.cineast.standalone.run.path; +import java.util.List; +import java.util.Optional; import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; -import java.util.List; -import java.util.Optional; - /** * Convenience Method when you don't want to actually provide Elements. - * */ public class NoContainerProvider implements ExtractionContainerProvider, ExtractionCompleteListener { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SessionContainerProvider.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SessionContainerProvider.java index 215d2c1ea..219326947 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SessionContainerProvider.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SessionContainerProvider.java @@ -2,26 +2,21 @@ import io.prometheus.client.Counter; import io.prometheus.client.Gauge; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; -import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; -import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; - import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.standalone.config.Config; +import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; +import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; +import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; /** - * A flexible Pathprovider with no caching. Simply stores a list of paths in memory. Differentiates - * between three states - running, closing and closed(=!open). This is necessary because when a - * Session is ended by the user, he still expects the submitted items to be extracted. Therefore, on - * an {@link #endSession()} call, the instance is only {@link #closing}, but still {@link #open}. - * + * A flexible Pathprovider with no caching. Simply stores a list of paths in memory. Differentiates between three states - running, closing and closed(=!open). This is necessary because when a Session is ended by the user, he still expects the submitted items to be extracted. Therefore, on an {@link #endSession()} call, the instance is only {@link #closing}, but still {@link #open}. */ public class SessionContainerProvider implements ExtractionContainerProvider, ExtractionCompleteListener { @@ -50,8 +45,7 @@ public SessionContainerProvider() { } /** - * Delayed close. After every item has been taken from the buffer, the instance will report itself - * as closed. + * Delayed close. After every item has been taken from the buffer, the instance will report itself as closed. */ public void endSession() { stateModification.lock(); @@ -128,8 +122,7 @@ public void onCompleted(ExtractionItemContainer path) { } /** - * Tells the container to not mark this instance as closing, no matter what the previous state - * was. + * Tells the container to not mark this instance as closing, no matter what the previous state was. */ public boolean keepAliveCheckIfClosed() { stateModification.lock(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SingletonContainerProvider.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SingletonContainerProvider.java index 70ea43bfe..52f91ff32 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SingletonContainerProvider.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/SingletonContainerProvider.java @@ -1,18 +1,16 @@ package org.vitrivr.cineast.standalone.run.path; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.standalone.config.IngestConfig; import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - /** * Convenience Provider for the {@link IngestConfig} - * */ public class SingletonContainerProvider implements ExtractionContainerProvider, ExtractionCompleteListener { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/TreeWalkContainerIteratorProvider.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/TreeWalkContainerIteratorProvider.java index 4caa1d9b7..8eb63d3f9 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/TreeWalkContainerIteratorProvider.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/path/TreeWalkContainerIteratorProvider.java @@ -1,16 +1,6 @@ package org.vitrivr.cineast.standalone.run.path; import io.prometheus.client.Counter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; -import org.vitrivr.cineast.core.extraction.ExtractionContextProvider; -import org.vitrivr.cineast.core.util.LogHelper; -import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; -import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; -import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; - import java.io.IOException; import java.nio.file.FileVisitOption; import java.nio.file.Files; @@ -19,6 +9,14 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; +import org.vitrivr.cineast.core.util.LogHelper; +import org.vitrivr.cineast.standalone.config.Config; +import org.vitrivr.cineast.standalone.run.ExtractionCompleteListener; +import org.vitrivr.cineast.standalone.run.ExtractionContainerProvider; +import org.vitrivr.cineast.standalone.run.ExtractionItemContainer; /* * Recursively add all files under that path to the List of files that should be processed. Uses @@ -51,7 +49,7 @@ public TreeWalkContainerIteratorProvider(Path basePath, Path startPath, int dept pathIterator = Files.walk(resolvedStartPath, this.depth, FileVisitOption.FOLLOW_LINKS) .filter(p -> { try { - return Files.exists(p) && !Files.isHidden(p) && Files.isReadable(p); + return Files.exists(p) && !Files.isHidden(p) && Files.isReadable(p); } catch (IOException e) { LOGGER.error("An IO exception occurred while testing the media file at '{}': {}", p.toString(), LogHelper.getStackTrace(e)); return false; @@ -77,8 +75,7 @@ public void addPaths(List pathList) { } /** - * Since no elements are added to the iterator, this provider is also closed when the iterator - * does not have further elements. + * Since no elements are added to the iterator, this provider is also closed when the iterator does not have further elements. */ @Override public boolean isOpen() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ContinuousQueryDispatcher.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ContinuousQueryDispatcher.java index 5bcb4d371..a1822be86 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ContinuousQueryDispatcher.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ContinuousQueryDispatcher.java @@ -3,6 +3,17 @@ import gnu.trove.iterator.TDoubleIterator; import gnu.trove.map.TObjectDoubleMap; import gnu.trove.map.hash.TObjectDoubleHashMap; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -22,11 +33,8 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.listener.RetrievalResultListener; -import java.util.*; -import java.util.concurrent.*; -import java.util.function.Function; - public class ContinuousQueryDispatcher { + private static final Logger LOGGER = LogManager.getLogger(); private static final String LISTENER_NULL_MESSAGE = "Retrieval result listener cannot be null."; @@ -132,7 +140,7 @@ private static void clearExecutor() { e.printStackTrace(); } List runnables = executor.shutdownNow(); - if(runnables.size()>0){ + if (runnables.size() > 0) { LOGGER.warn("{} threads terminated prematurely", runnables.size()); } executor = null; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExecutionTimeCounter.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExecutionTimeCounter.java index 1d52934e8..ef6e1b73e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExecutionTimeCounter.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExecutionTimeCounter.java @@ -4,7 +4,8 @@ public interface ExecutionTimeCounter { /** * used to report task execution time for a particular class - * @param name classname. is a string since we can't determine the classname of generic types at runtime + * + * @param name classname. is a string since we can't determine the classname of generic types at runtime * @param miliseconds the task duration in ms */ void reportExecutionTime(String name, long miliseconds); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java index c3a02451a..2be8bf4fe 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java @@ -1,5 +1,13 @@ package org.vitrivr.cineast.standalone.runtime; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -11,226 +19,228 @@ import org.vitrivr.cineast.standalone.config.ExtractionPipelineConfig; import org.vitrivr.cineast.standalone.monitoring.PrometheusExtractionTaskMonitor; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.*; - public class ExtractionPipeline implements Runnable, ExecutionTimeCounter { - private static final Logger LOGGER = LogManager.getLogger(); - - /** The list of Extractor's that should be executed. */ - private final List extractors = new LinkedList<>(); - - /** Blocking queue holding the SegmentContainers that are pending extraction. */ - private final LinkedBlockingQueue segmentQueue; - - /** HashMap containing statistics about the execution of the extractors. */ - private final ConcurrentHashMap timeMap = new ConcurrentHashMap<>(); - - /** ExecutorService used do execute the ExtractionTasks. */ - private final ExecutorService executorService; - - /** ExtractionContextProvider used to setup the Pipeline. It contains information about the Extractors. */ - private final ExtractionContextProvider context; - - /** Flag indicating whether or not the ExtractionPipeline is running. */ - private volatile boolean running = false; - - /** - * Default constructor. - * - * @param context ExtractionContextProvider used to setup the pipeline. - */ - public ExtractionPipeline(ExtractionContextProvider context) { - /* Store context for further reference. */ - this.context = context; - - /* Start the extraction pipeline. */ - this.startup(); - - /* Get value for size task-queue and number of threads. */ - int taskQueueSize = context.taskQueueSize() > 0 ? context.taskQueueSize() : ExtractionPipelineConfig.DEFAULT_TASKQUEUE_SIZE; - int threadCount = context.threadPoolSize() > 0 ? context.threadPoolSize() : ExtractionPipelineConfig.DEFAULT_THREADPOOL_SIZE; - int segmentQueueSize = context.segmentQueueSize() > 0 ? context.segmentQueueSize() : ExtractionPipelineConfig.DEFAULT_SEGMENTQUEUE_SIZE; - - /* Initialize the segment queue. */ - this.segmentQueue = new LinkedBlockingQueue<>(segmentQueueSize); - - /* Prepare and create a new ThreadPoolExecutor. */ - LimitedQueue taskQueue = new LimitedQueue<>(taskQueueSize); - this.executorService = new ThreadPoolExecutor(threadCount, threadCount, 60, TimeUnit.SECONDS, taskQueue){ - @Override - protected void afterExecute(Runnable r, Throwable t) { - if(t != null){ - LOGGER.fatal("Decoding Error detected!"); - LOGGER.fatal(LogHelper.getStackTrace(t)); - //this.shutdownNow(); - } - super.afterExecute(r, null); - } - }; - } - - /** - * Sets the running flag to false which will halt the execution of the ExtractionPipeline as soon - * as possible. Even when halted, there might still be ExtractionTasks in the executor that have not - * finished yet! - */ - public synchronized void stop() { - this.running = false; - } - - /** - * Indicates whether or not the ExtractionPipeline is still running i.e. the while loop in the run() - * method is still being executed. Even if this flag is set to false, there might still be ExtractionTasks - * in the executor that have not finished yet! - * - * @return True if the ExtractionPipeline is running, false otherwise. - */ - public synchronized boolean isRunning() { - return this.running; - } - - /** - * Can be used to emit a SegmentContainer into the ExtractionPipeline. Invoking this method will add - * the container to the local queue. - * - * If that queue is full, this method will block for the specified amount of time or until - * space to becomes available. If during that time, no space becomes available, the method - * returns false. This is an indication that emission of the segment should be retried later. - * - * @param container SegmentContainer to add to the queue. - * @param timeout Time to wait for space to become available in ms. - * @return true if SegmentContainer was emitted, false otherwise. - */ - public boolean emit(SegmentContainer container, int timeout) throws InterruptedException { - return this.segmentQueue.offer(container, timeout, TimeUnit.MILLISECONDS); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * The list of Extractor's that should be executed. + */ + private final List extractors = new LinkedList<>(); + + /** + * Blocking queue holding the SegmentContainers that are pending extraction. + */ + private final LinkedBlockingQueue segmentQueue; + + /** + * HashMap containing statistics about the execution of the extractors. + */ + private final ConcurrentHashMap timeMap = new ConcurrentHashMap<>(); + + /** + * ExecutorService used do execute the ExtractionTasks. + */ + private final ExecutorService executorService; + + /** + * ExtractionContextProvider used to setup the Pipeline. It contains information about the Extractors. + */ + private final ExtractionContextProvider context; + + /** + * Flag indicating whether or not the ExtractionPipeline is running. + */ + private volatile boolean running = false; + + /** + * Default constructor. + * + * @param context ExtractionContextProvider used to setup the pipeline. + */ + public ExtractionPipeline(ExtractionContextProvider context) { + /* Store context for further reference. */ + this.context = context; + + /* Start the extraction pipeline. */ + this.startup(); + + /* Get value for size task-queue and number of threads. */ + int taskQueueSize = context.taskQueueSize() > 0 ? context.taskQueueSize() : ExtractionPipelineConfig.DEFAULT_TASKQUEUE_SIZE; + int threadCount = context.threadPoolSize() > 0 ? context.threadPoolSize() : ExtractionPipelineConfig.DEFAULT_THREADPOOL_SIZE; + int segmentQueueSize = context.segmentQueueSize() > 0 ? context.segmentQueueSize() : ExtractionPipelineConfig.DEFAULT_SEGMENTQUEUE_SIZE; + + /* Initialize the segment queue. */ + this.segmentQueue = new LinkedBlockingQueue<>(segmentQueueSize); + + /* Prepare and create a new ThreadPoolExecutor. */ + LimitedQueue taskQueue = new LimitedQueue<>(taskQueueSize); + this.executorService = new ThreadPoolExecutor(threadCount, threadCount, 60, TimeUnit.SECONDS, taskQueue) { + @Override + protected void afterExecute(Runnable r, Throwable t) { + if (t != null) { + LOGGER.fatal("Decoding Error detected!"); + LOGGER.fatal(LogHelper.getStackTrace(t)); + //this.shutdownNow(); + } + super.afterExecute(r, null); + } + }; + } + + /** + * Sets the running flag to false which will halt the execution of the ExtractionPipeline as soon as possible. Even when halted, there might still be ExtractionTasks in the executor that have not finished yet! + */ + public synchronized void stop() { + this.running = false; + } + + /** + * Indicates whether or not the ExtractionPipeline is still running i.e. the while loop in the run() method is still being executed. Even if this flag is set to false, there might still be ExtractionTasks in the executor that have not finished yet! + * + * @return True if the ExtractionPipeline is running, false otherwise. + */ + public synchronized boolean isRunning() { + return this.running; + } + + /** + * Can be used to emit a SegmentContainer into the ExtractionPipeline. Invoking this method will add the container to the local queue. + *

    + * If that queue is full, this method will block for the specified amount of time or until space to becomes available. If during that time, no space becomes available, the method returns false. This is an indication that emission of the segment should be retried later. + * + * @param container SegmentContainer to add to the queue. + * @param timeout Time to wait for space to become available in ms. + * @return true if SegmentContainer was emitted, false otherwise. + */ + public boolean emit(SegmentContainer container, int timeout) throws InterruptedException { + return this.segmentQueue.offer(container, timeout, TimeUnit.MILLISECONDS); + } + + /** + * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's + * run method to be called in that separately executing + * thread. + *

    + * The general contract of the method run is that it may take any action whatsoever. + * + * @see Thread#run() + */ + @Override + public void run() { + /* Set running flag to true. */ + synchronized (this) { + this.running = true; } - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

    - * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - /* Set running flag to true. */ - synchronized (this) { this.running = true; } - - /* Process SegmentContainers in Queue: For each Extractor in list dispatch an extraction task. */ - while (this.isRunning() || !this.segmentQueue.isEmpty()) { - if(!this.isRunning()){ - LOGGER.debug("Received stop signal, still {} elements left", this.segmentQueue.size()); - } + /* Process SegmentContainers in Queue: For each Extractor in list dispatch an extraction task. */ + while (this.isRunning() || !this.segmentQueue.isEmpty()) { + if (!this.isRunning()) { + LOGGER.debug("Received stop signal, still {} elements left", this.segmentQueue.size()); + } + try { + SegmentContainer s = this.segmentQueue.poll(500, TimeUnit.MILLISECONDS); + if (s != null) { + LOGGER.debug("Segment {} is being handed to the extraction pipeline.", s.getId()); + for (Extractor f : extractors) { try { - SegmentContainer s = this.segmentQueue.poll(500, TimeUnit.MILLISECONDS); - if (s != null) { - LOGGER.debug("Segment {} is being handed to the extraction pipeline.", s.getId()); - for (Extractor f : extractors) { - try { - this.executorService.execute(new ExtractionTask(f, s, this)); - LOGGER.debug("Submitted segment {} for feature {}", s.getId(), f.getClass().getSimpleName()); - } catch (RejectedExecutionException e) { - this.segmentQueue.clear(); - LOGGER.fatal("Failed to submit segment {} for feature {}. Aborting...\n{}", s.getId(), f.getClass().getSimpleName(), LogHelper.getStackTrace(e)); - break; - } - } - - /* Sort list of extractors by execution time. */ - (this.extractors).sort((o1,o2) -> Long.compare(getAverageExecutionTime(o2.getClass().getSimpleName()), getAverageExecutionTime(o1.getClass().getSimpleName()))); - } - } catch (InterruptedException e) { - LOGGER.warn("ShotDispatcher was interrupted: {}", LogHelper.getStackTrace(e)); - } - } - - this.shutdown(); + this.executorService.execute(new ExtractionTask(f, s, this)); + LOGGER.debug("Submitted segment {} for feature {}", s.getId(), f.getClass().getSimpleName()); + } catch (RejectedExecutionException e) { + this.segmentQueue.clear(); + LOGGER.fatal("Failed to submit segment {} for feature {}. Aborting...\n{}", s.getId(), f.getClass().getSimpleName(), LogHelper.getStackTrace(e)); + break; + } + } - synchronized (this) { this.running = false; } + /* Sort list of extractors by execution time. */ + (this.extractors).sort((o1, o2) -> Long.compare(getAverageExecutionTime(o2.getClass().getSimpleName()), getAverageExecutionTime(o1.getClass().getSimpleName()))); + } + } catch (InterruptedException e) { + LOGGER.warn("ShotDispatcher was interrupted: {}", LogHelper.getStackTrace(e)); + } } - /** - * Starts the ExtractionPipeline by initializing the Extractors. - */ - private void startup() { - LOGGER.info("Warming up extraction pipeline...."); + this.shutdown(); - for (Extractor extractor : this.context.extractors()) { - try { - extractor.init(this.context.persistencyWriter(), this.context.batchSize()); - this.extractors.add(extractor); - } catch (Throwable t) { - LOGGER.warn("Failed to initialize extractor {} due to an exception: {}", extractor.getClass().getSimpleName(), t.getMessage()); - } - } + synchronized (this) { + this.running = false; + } + } + + /** + * Starts the ExtractionPipeline by initializing the Extractors. + */ + private void startup() { + LOGGER.info("Warming up extraction pipeline...."); + + for (Extractor extractor : this.context.extractors()) { + try { + extractor.init(this.context.persistencyWriter(), this.context.batchSize()); + this.extractors.add(extractor); + } catch (Throwable t) { + LOGGER.warn("Failed to initialize extractor {} due to an exception: {}", extractor.getClass().getSimpleName(), t.getMessage()); + } + } - for (Extractor exporter : this.context.exporters()) { - try { - exporter.init(this.context.persistencyWriter(), this.context.batchSize()); - this.extractors.add(exporter); - } catch (Throwable t) { - LOGGER.warn("Failed to exporter extractor {} due to an exception: {}", exporter.getClass().getSimpleName(), t.getMessage()); - } - } - - LOGGER.info("Extraction pipeline is ready with {} extractors.", this.extractors.size()); + for (Extractor exporter : this.context.exporters()) { + try { + exporter.init(this.context.persistencyWriter(), this.context.batchSize()); + this.extractors.add(exporter); + } catch (Throwable t) { + LOGGER.warn("Failed to exporter extractor {} due to an exception: {}", exporter.getClass().getSimpleName(), t.getMessage()); + } } - /** - * Shuts the ExtractionPipeline down. Stops the ExecutorService and finishes the - * extractors. - */ - private void shutdown() { + LOGGER.info("Extraction pipeline is ready with {} extractors.", this.extractors.size()); + } + + /** + * Shuts the ExtractionPipeline down. Stops the ExecutorService and finishes the extractors. + */ + private void shutdown() { + try { + LOGGER.debug("Shutting down executor service"); + this.executorService.shutdown(); + LOGGER.debug("Waiting for termination of executor"); + this.executorService.awaitTermination(15, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while waiting for Executor to shut down!"); + } finally { + for (Extractor extractor : this.extractors) { try { - LOGGER.debug("Shutting down executor service"); - this.executorService.shutdown(); - LOGGER.debug("Waiting for termination of executor"); - this.executorService.awaitTermination(15, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting for Executor to shut down!"); - } finally { - for (Extractor extractor : this.extractors) { - try { - extractor.finish(); - } catch (Exception e) { - LOGGER.error("Error while shutting down extractor {} : {}", - extractor.getClass().getSimpleName(), LogHelper.getStackTrace(e)); - } - } - LOGGER.debug("All extractors termination, executor service terminated. Exiting shutdown."); + extractor.finish(); + } catch (Exception e) { + LOGGER.error("Error while shutting down extractor {} : {}", + extractor.getClass().getSimpleName(), LogHelper.getStackTrace(e)); } + } + LOGGER.debug("All extractors termination, executor service terminated. Exiting shutdown."); } + } - @Override - public void reportExecutionTime(String className, long milliseconds) { - if(!this.timeMap.containsKey(className)){ - this.timeMap.put(className, new SummaryStatistics()); - } - SummaryStatistics stat = this.timeMap.get(className); - synchronized (stat) { - stat.addValue(milliseconds); - } - PrometheusExtractionTaskMonitor.reportExecutionTime(className, milliseconds); + @Override + public void reportExecutionTime(String className, long milliseconds) { + if (!this.timeMap.containsKey(className)) { + this.timeMap.put(className, new SummaryStatistics()); } - - /** - * @return the average execution time for all tasks reported for this class or 0 if the class is unknown or null - */ - @Override - public long getAverageExecutionTime(String className) { - if(this.timeMap.containsKey(className)){ - return (long) this.timeMap.get(className).getMean(); - } - return 0; + SummaryStatistics stat = this.timeMap.get(className); + synchronized (stat) { + stat.addValue(milliseconds); + } + PrometheusExtractionTaskMonitor.reportExecutionTime(className, milliseconds); + } + + /** + * @return the average execution time for all tasks reported for this class or 0 if the class is unknown or null + */ + @Override + public long getAverageExecutionTime(String className) { + if (this.timeMap.containsKey(className)) { + return (long) this.timeMap.get(className).getMean(); } + return 0; + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/RetrievalTask.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/RetrievalTask.java index e154d3018..7f8904131 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/RetrievalTask.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/RetrievalTask.java @@ -64,8 +64,8 @@ public Pair> call() throws Exception { private void nameThread() { String currentThreadName = Thread.currentThread().getName(); - if(!currentThreadName.endsWith(retriever.getClass().getSimpleName())){ - Thread.currentThread().setName(currentThreadName.substring(0, currentThreadName.lastIndexOf('-'))+"-"+retriever.getClass().getSimpleName()); + if (!currentThreadName.endsWith(retriever.getClass().getSimpleName())) { + Thread.currentThread().setName(currentThreadName.substring(0, currentThreadName.lastIndexOf('-')) + "-" + retriever.getClass().getSimpleName()); } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/CLI.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/CLI.java index 8b5a78fb4..c02865444 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/CLI.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/CLI.java @@ -2,6 +2,13 @@ import com.github.rvesse.airline.model.CommandMetadata; import com.github.rvesse.airline.parser.errors.ParseRestrictionViolatedException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.jline.builtins.Completers; import org.jline.reader.Completer; import org.jline.reader.EndOfFileException; @@ -14,120 +21,109 @@ import org.jline.utils.AttributedStringBuilder; import org.jline.utils.AttributedStyle; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - /** * Helper class that can be used to start an interactive CLI. - * */ public class CLI { - private static final String PROMPT = "cineast> "; + private static final String PROMPT = "cineast> "; - private static final String LOGO = "################################################################################\u0085#                                                                              #\u0085#                @@@                           @@@                             #\u0085#                @@@                           @@@                             #\u0085#                     @@@@                                                     #\u0085#   @@@     @@@  @@@  @@@@@@@         @@@@@    @@@  @@@     @@@     @@@@@      #\u0085#   @@@@   @@@@  @@@  @@@@          @@@@@@@@@  @@@  @@@@   @@@@   @@@@@@@@@    #\u0085#     @@@@@@@    @@@  @@@@    @@@  @@@@        @@@    @@@@@@@    @@@@          #\u0085#      @@@@@     @@@   @@@@@@@@@@  @@@@        @@@     @@@@@     @@@@          #\u0085#       @@@      @@@     @@@@@     @@@         @@@      @@@      @@@           #\u0085#                                                                              #\u0085################################################################################"; + private static final String LOGO = "################################################################################\u0085#                                                                              #\u0085#                @@@                           @@@                             #\u0085#                @@@                           @@@                             #\u0085#                     @@@@                                                     #\u0085#   @@@     @@@  @@@  @@@@@@@         @@@@@    @@@  @@@     @@@     @@@@@      #\u0085#   @@@@   @@@@  @@@  @@@@          @@@@@@@@@  @@@  @@@@   @@@@   @@@@@@@@@    #\u0085#     @@@@@@@    @@@  @@@@    @@@  @@@@        @@@    @@@@@@@    @@@@          #\u0085#      @@@@@     @@@   @@@@@@@@@@  @@@@        @@@     @@@@@     @@@@          #\u0085#       @@@      @@@     @@@@@     @@@         @@@      @@@      @@@           #\u0085#                                                                              #\u0085################################################################################"; - private CLI() { - } + private CLI() { + } - /** - * Starts the interactive CLI. This is method will block. - */ - public static void start(Class cliClass) { + /** + * Starts the interactive CLI. This is method will block. + */ + public static void start(Class cliClass) { - Terminal terminal = null; - try { - terminal = TerminalBuilder.terminal(); //basic terminal - } catch (IOException e) { - System.err.println("Could not initialize Terminal: "); - System.err.println(e.getMessage()); - System.err.println("Exiting..."); - System.exit(-1); - } + Terminal terminal = null; + try { + terminal = TerminalBuilder.terminal(); //basic terminal + } catch (IOException e) { + System.err.println("Could not initialize Terminal: "); + System.err.println(e.getMessage()); + System.err.println("Exiting..."); + System.exit(-1); + } - final com.github.rvesse.airline.Cli cli = new com.github.rvesse.airline.Cli<>(cliClass); - final List commandNames = cli.getMetadata().getDefaultGroupCommands().stream() - .map(CommandMetadata::getName).collect(Collectors.toList()); + final com.github.rvesse.airline.Cli cli = new com.github.rvesse.airline.Cli<>(cliClass); + final List commandNames = cli.getMetadata().getDefaultGroupCommands().stream() + .map(CommandMetadata::getName).collect(Collectors.toList()); - - final Completer completer = new AggregateCompleter( - new StringsCompleter("quit", "exit", "stop"), - new StringsCompleter(commandNames), - new Completers.FileNameCompleter() - ); + final Completer completer = new AggregateCompleter( + new StringsCompleter("quit", "exit", "stop"), + new StringsCompleter(commandNames), + new Completers.FileNameCompleter() + ); - final LineReader lineReader = LineReaderBuilder.builder() - .terminal(terminal) - .completer(completer) - .build(); + final LineReader lineReader = LineReaderBuilder.builder() + .terminal(terminal) + .completer(completer) + .build(); - terminal.writer().println(LOGO.replaceAll("\u0085", "\r\n")); - terminal.writer().println("Welcome to the interactive Cineast CLI."); + terminal.writer().println(LOGO.replaceAll("\u0085", "\r\n")); + terminal.writer().println("Welcome to the interactive Cineast CLI."); + try { + while (true) { + final String line; + /* Catch ^D end of file as exit method */ try { - while (true) { - - final String line; - /* Catch ^D end of file as exit method */ - try { - line = lineReader.readLine(PROMPT).trim(); - } catch (EndOfFileException e) { - break; - } - - if (line.toLowerCase().equals("exit") || line.toLowerCase().equals("quit") || line.toLowerCase().equals("stop")) { - break; - } - - /* Try to parse user input. */ - try { - final Runnable command = cli.parse(CLI.splitLine(line)); - command.run(); - } catch (ParseRestrictionViolatedException e) { - terminal.writer().println( - new AttributedStringBuilder().style(AttributedStyle.DEFAULT.foreground(AttributedStyle.RED)) - .append("Error: ").append(e.getMessage()).toAnsi() - ); - } catch (Exception e) { - terminal.writer().println( - new AttributedStringBuilder().style(AttributedStyle.DEFAULT.foreground(AttributedStyle.RED)) - .append("Error: ").append(e.getMessage()).toAnsi() - ); - } - } - } catch (IllegalStateException | NoSuchElementException e) { - System.out.println("System.in was closed; exiting"); + line = lineReader.readLine(PROMPT).trim(); + } catch (EndOfFileException e) { + break; } - } - - final static Pattern lineSplitRegex = Pattern.compile("[^\\s\"']+|\"([^\"]*)\"|'([^']*)'"); - //based on https://stackoverflow.com/questions/366202/regex-for-splitting-a-string-using-space-when-not-surrounded-by-single-or-double/366532 - private static String[] splitLine(String line){ - if (line == null || line.isEmpty()){ - return new String[0]; + if (line.toLowerCase().equals("exit") || line.toLowerCase().equals("quit") || line.toLowerCase().equals("stop")) { + break; } - List matchList = new ArrayList(); - Matcher regexMatcher = lineSplitRegex.matcher(line); - while (regexMatcher.find()) { - if (regexMatcher.group(1) != null) { - // Add double-quoted string without the quotes - matchList.add(regexMatcher.group(1)); - } else if (regexMatcher.group(2) != null) { - // Add single-quoted string without the quotes - matchList.add(regexMatcher.group(2)); - } else { - // Add unquoted word - matchList.add(regexMatcher.group()); - } + + /* Try to parse user input. */ + try { + final Runnable command = cli.parse(CLI.splitLine(line)); + command.run(); + } catch (ParseRestrictionViolatedException e) { + terminal.writer().println( + new AttributedStringBuilder().style(AttributedStyle.DEFAULT.foreground(AttributedStyle.RED)) + .append("Error: ").append(e.getMessage()).toAnsi() + ); + } catch (Exception e) { + terminal.writer().println( + new AttributedStringBuilder().style(AttributedStyle.DEFAULT.foreground(AttributedStyle.RED)) + .append("Error: ").append(e.getMessage()).toAnsi() + ); } - return matchList.toArray(new String[matchList.size()]); + } + } catch (IllegalStateException | NoSuchElementException e) { + System.out.println("System.in was closed; exiting"); + } + } + + final static Pattern lineSplitRegex = Pattern.compile("[^\\s\"']+|\"([^\"]*)\"|'([^']*)'"); + + //based on https://stackoverflow.com/questions/366202/regex-for-splitting-a-string-using-space-when-not-surrounded-by-single-or-double/366532 + private static String[] splitLine(String line) { + if (line == null || line.isEmpty()) { + return new String[0]; + } + List matchList = new ArrayList(); + Matcher regexMatcher = lineSplitRegex.matcher(line); + while (regexMatcher.find()) { + if (regexMatcher.group(1) != null) { + // Add double-quoted string without the quotes + matchList.add(regexMatcher.group(1)); + } else if (regexMatcher.group(2) != null) { + // Add single-quoted string without the quotes + matchList.add(regexMatcher.group(2)); + } else { + // Add unquoted word + matchList.add(regexMatcher.group()); + } } + return matchList.toArray(new String[matchList.size()]); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/ContinuousRetrievalLogic.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/ContinuousRetrievalLogic.java index 7c51b3daf..f2cafc2e5 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/ContinuousRetrievalLogic.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/util/ContinuousRetrievalLogic.java @@ -1,6 +1,9 @@ package org.vitrivr.cineast.standalone.util; import gnu.trove.map.hash.TObjectDoubleHashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; @@ -14,10 +17,6 @@ import org.vitrivr.cineast.standalone.listener.RetrievalResultListener; import org.vitrivr.cineast.standalone.runtime.ContinuousQueryDispatcher; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - public class ContinuousRetrievalLogic { private static final Logger LOGGER = LogManager.getLogger(); @@ -25,7 +24,7 @@ public class ContinuousRetrievalLogic { private final RetrieverInitializer initializer; private final MediaSegmentReader segmentReader; - public ContinuousRetrievalLogic(DatabaseConfig config){ + public ContinuousRetrievalLogic(DatabaseConfig config) { this.config = config; this.initializer = r -> r.init(this.config.getSelectorSupplier()); this.segmentReader = new MediaSegmentReader(this.config.getSelectorSupplier().get()); diff --git a/cineast-runtime/src/main/resources/log4j2.xml b/cineast-runtime/src/main/resources/log4j2.xml index acf111848..d78f80b5d 100644 --- a/cineast-runtime/src/main/resources/log4j2.xml +++ b/cineast-runtime/src/main/resources/log4j2.xml @@ -1,34 +1,34 @@ - - [%d{MM-dd HH:mm:ss.SSS}][%-5level][%t] %C{1} - %msg%n - - - - - - - - logs/cineast.log - logs/%d{yyyy-MM-dd-hh}-%i.log.zip - - - - - - - - - - - - - - - - - - - - + + [%d{MM-dd HH:mm:ss.SSS}][%-5level][%t] %C{1} - %msg%n + + + + + + + + logs/cineast.log + logs/%d{yyyy-MM-dd-hh}-%i.log.zip + + + + + + + + + + + + + + + + + + + + From cea3083444273728784ca19a2ea96ff3a7979ede Mon Sep 17 00:00:00 2001 From: Florian Spiess Date: Tue, 4 Jan 2022 10:13:47 +0100 Subject: [PATCH 12/72] Fixed CottontailWrapper pretending connection was successful even when it wasn't. --- .../cineast/core/db/cottontaildb/CottontailWrapper.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 92b39465d..65a83585d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -51,9 +51,14 @@ public CottontailWrapper(DatabaseConfig config, boolean keepOpen) { } this.channel = builder.build(); this.client = new SimpleClient(this.channel); + + boolean pingSuccessful = this.client.ping(); watch.stop(); - LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), - config.getHost(), config.getPort()); + if (pingSuccessful) { + LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), config.getHost(), config.getPort()); + } else { + LOGGER.warn("Could not connect to Cottontail at {}:{}", config.getHost(), config.getPort()); + } } /** From fa880b8256c1fcbb94a90af859a9bf72a3ec5a70 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Fri, 21 Jan 2022 20:17:09 +0100 Subject: [PATCH 13/72] Added base structure for skeleton pose feature --- .../org/vitrivr/cineast/core/data/Grid.java | 1 + .../vitrivr/cineast/core/data/Position.java | 1 + .../vitrivr/cineast/core/data/Skeleton.java | 73 ++++++++++++++ .../cineast/core/data/UniqueElementGrid.java | 2 + .../core/data/providers/SkeletonProvider.java | 14 +++ .../core/data/segments/SegmentContainer.java | 59 ++++------- .../cineast/core/features/SkeletonPose.java | 99 +++++++++++++++++++ 7 files changed, 211 insertions(+), 38 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java index 4cefa6ffc..aea4481e8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Grid.java @@ -2,6 +2,7 @@ import java.util.Set; +@Deprecated public interface Grid { Grid setElement(int x, int y, T element); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java index e91355589..7aff91298 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Position.java @@ -5,6 +5,7 @@ /** * Created by silvanstich on 04.10.16. */ +@Deprecated public class Position implements Serializable { private static final long serialVersionUID = -4360148820779089159L; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java new file mode 100644 index 000000000..a26c6dee4 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -0,0 +1,73 @@ +package org.vitrivr.cineast.core.data; + +import georegression.struct.point.Point2D_F32; + +import java.util.ArrayList; +import java.util.List; + +public class Skeleton { + + public enum SkeletonPointName { + NOSE, + LEFT_EYE, + RIGHT_EYE, + LEFT_EAR, + RIGHT_EAR, + LEFT_SHOULDER, + RIGHT_SHOULDER, + LEFT_ELBOW, + RIGHT_ELBOW, + LEFT_WRIST, + RIGHT_WRIST, + LEFT_HIP, + RIGHT_HIP, + LEFT_KNEE, + RIGHT_KNEE, + LEFT_ANKLE, + RIGHT_ANKLE + } + + /* + + [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], [5, 11], [6, 12], [5, 6], [5, 7], + [6, 8], [7, 9], [8, 10], [1, 2], [0, 1], [0, 2], [1, 3], [2, 4], + [0, 5], [0, 6] + + */ + + private static final int POINT_COUNT = 17; + + private final Point2D_F32[] points = new Point2D_F32[POINT_COUNT]; + private final float[] weights = new float[POINT_COUNT]; + + public Skeleton(float[] coordinates, float[] weights) { + + if (coordinates == null || coordinates.length < 2 * POINT_COUNT || weights == null || weights.length < POINT_COUNT) { + throw new IllegalArgumentException(); + } + + for (int i = 0; i < POINT_COUNT; ++i) { + this.weights[i] = weights[i]; + this.points[i] = new Point2D_F32(coordinates[2 * i], coordinates[2 * i + 1]); + } + + } + + public List getPoints() { + ArrayList _return = new ArrayList<>(POINT_COUNT); + for (int i = 0; i < POINT_COUNT; ++i) { + _return.add(this.points[i].copy()); + } + return _return; + } + + public Point2D_F32 getPoint(SkeletonPointName name) { + return this.points[name.ordinal()].copy(); + } + + public float getWeight(SkeletonPointName name) { + return this.weights[name.ordinal()]; + } + + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java index 2252aed03..4045c0e10 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/UniqueElementGrid.java @@ -1,9 +1,11 @@ package org.vitrivr.cineast.core.data; import com.google.common.collect.HashBiMap; + import java.util.HashSet; import java.util.Set; +@Deprecated public class UniqueElementGrid implements Grid { private final HashBiMap map = HashBiMap.create(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java new file mode 100644 index 000000000..40efd472b --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java @@ -0,0 +1,14 @@ +package org.vitrivr.cineast.core.data.providers; + +import org.vitrivr.cineast.core.data.Skeleton; + +import java.util.Collections; +import java.util.List; + +public interface SkeletonProvider { + + default List getSkeletons() { + return Collections.emptyList(); + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java index 74cfb272a..54bac1e3b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java @@ -1,23 +1,6 @@ package org.vitrivr.cineast.core.data.segments; -import org.vitrivr.cineast.core.data.providers.AudioFrameProvider; -import org.vitrivr.cineast.core.data.providers.AudioSTFTProvider; -import org.vitrivr.cineast.core.data.providers.AvgImgProvider; -import org.vitrivr.cineast.core.data.providers.BooleanExpressionProvider; -import org.vitrivr.cineast.core.data.providers.DurationProvider; -import org.vitrivr.cineast.core.data.providers.FrameListProvider; -import org.vitrivr.cineast.core.data.providers.IdProvider; -import org.vitrivr.cineast.core.data.providers.InstantProvider; -import org.vitrivr.cineast.core.data.providers.LocationProvider; -import org.vitrivr.cineast.core.data.providers.MedianImgProvider; -import org.vitrivr.cineast.core.data.providers.MeshProvider; -import org.vitrivr.cineast.core.data.providers.MostRepresentativeFrameProvider; -import org.vitrivr.cineast.core.data.providers.PathProvider; -import org.vitrivr.cineast.core.data.providers.SemanticMapProvider; -import org.vitrivr.cineast.core.data.providers.SubtitleItemProvider; -import org.vitrivr.cineast.core.data.providers.TagProvider; -import org.vitrivr.cineast.core.data.providers.TextProvider; -import org.vitrivr.cineast.core.data.providers.VoxelGridProvider; +import org.vitrivr.cineast.core.data.providers.*; import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.features.retriever.Retriever; @@ -27,23 +10,23 @@ * During the offline phase, it is passed to an {@link Extractor}, and during the online phase, it is passed to a {@link Retriever}. */ public interface SegmentContainer - extends IdProvider, - AvgImgProvider, - DurationProvider, - MedianImgProvider, - MostRepresentativeFrameProvider, - SubtitleItemProvider, - PathProvider, - TagProvider, - FrameListProvider, - AudioFrameProvider, - AudioSTFTProvider, - MeshProvider, - VoxelGridProvider, - LocationProvider, - InstantProvider, - TextProvider, - SemanticMapProvider, - BooleanExpressionProvider { - -} + extends IdProvider, + AvgImgProvider, + DurationProvider, + MedianImgProvider, + MostRepresentativeFrameProvider, + SubtitleItemProvider, + PathProvider, + TagProvider, + FrameListProvider, + AudioFrameProvider, + AudioSTFTProvider, + MeshProvider, + VoxelGridProvider, + LocationProvider, + InstantProvider, + TextProvider, + SemanticMapProvider, + BooleanExpressionProvider, + SkeletonProvider +{} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java new file mode 100644 index 000000000..f4fbd8fee --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -0,0 +1,99 @@ +package org.vitrivr.cineast.core.features; + +import georegression.struct.point.Point2D_F32; +import org.apache.commons.lang3.NotImplementedException; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.Pair; +import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +public class SkeletonPose extends AbstractFeatureModule { + + public SkeletonPose() { + super("feature_skeletonpose", 1, 20); + } + + @Override + public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + super.init(phandlerSupply, batchSize); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "person", "skeleton", "weights"); + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(this.tableName, false, + new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), + new AttributeDefinition("person", AttributeDefinition.AttributeType.INT), + new AttributeDefinition("skeleton", AttributeDefinition.AttributeType.VECTOR, 20), + new AttributeDefinition("weights", AttributeDefinition.AttributeType.VECTOR, 20) + ); + } + + @Override + public void processSegment(SegmentContainer segmentContainer) { + throw new NotImplementedException("not currently available"); + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + + List skeletons = sc.getSkeletons(); + + if (skeletons == null || skeletons.isEmpty()) { + return Collections.emptyList(); + } + + //TODO query + + return null; + } + + private Pair getAnglesandWeights(Skeleton skeleton) { + float[] angles = new float[]{ + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ANKLE),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ANKLE)), + + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_WRIST),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_WRIST)) + }; + + float[] weights = new float[]{ + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ANKLE),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ANKLE)), + + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_WRIST),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_WRIST)), + }; + + return new Pair<>(angles, weights); + + } + + private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { + return (float) (Math.atan2(p2.y - c.y, p2.x - c.x) - Math.atan2(p1.y - c.y, p1.x - c.x)); + } + + private static float min(float f, float g, float h) { + return Math.min(f, Math.min(g, h)); + } + +} From cb2e51ac2bd1c3ed80d379782366daf58b4214a5 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 24 Jan 2022 14:10:02 +0100 Subject: [PATCH 14/72] Implemented first version of skeleton retrieval logic --- .../vitrivr/cineast/core/data/Skeleton.java | 8 ++ .../cineast/core/features/SkeletonPose.java | 114 ++++++++++++++++-- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java index a26c6dee4..a0af43ca8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -61,6 +61,14 @@ public List getPoints() { return _return; } + public List getPointsScaled(float scaleX, float scaleY) { + ArrayList _return = new ArrayList<>(POINT_COUNT); + for (int i = 0; i < POINT_COUNT; ++i) { + _return.add(new Point2D_F32(points[i].x * scaleX, points[i].y * scaleY)); + } + return _return; + } + public Point2D_F32 getPoint(SkeletonPointName name) { return this.points[name.ordinal()].copy(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index f4fbd8fee..532c90f0c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -1,42 +1,53 @@ package org.vitrivr.cineast.core.features; import georegression.struct.point.Point2D_F32; +import gnu.trove.map.hash.TIntIntHashMap; +import gnu.trove.map.hash.TObjectDoubleHashMap; import org.apache.commons.lang3.NotImplementedException; +import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.score.SegmentScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; +import org.vitrivr.cineast.core.util.HungarianAlgorithm; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.function.Supplier; +import java.util.stream.Collectors; +import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; public class SkeletonPose extends AbstractFeatureModule { + private static final String PERSON_ID_COL = "person"; + private static final String FEATURE_COL = "skeleton"; + private static final String WEIGHT_COL = "weights"; + public SkeletonPose() { - super("feature_skeletonpose", 1, 20); + super("feature_skeletonpose", 1, 8); } @Override public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { super.init(phandlerSupply, batchSize); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "person", "skeleton", "weights"); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, PERSON_ID_COL, FEATURE_COL, WEIGHT_COL); } @Override public void initalizePersistentLayer(Supplier supply) { supply.get().createFeatureEntity(this.tableName, false, new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), - new AttributeDefinition("person", AttributeDefinition.AttributeType.INT), - new AttributeDefinition("skeleton", AttributeDefinition.AttributeType.VECTOR, 20), - new AttributeDefinition("weights", AttributeDefinition.AttributeType.VECTOR, 20) + new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), + new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), + new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) ); } @@ -54,9 +65,94 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc return Collections.emptyList(); } - //TODO query + HashMap>> segmentDistancesMap = new HashMap<>(qc.getRawResultsPerModule() * skeletons.size()); + + int queryPersonId = 0; + + //query all skeletons + for (Skeleton skeleton : skeletons) { + + Pair pair = getAnglesandWeights(skeleton); + + List> rows = this.selector.getNearestNeighbourRows(qc.getRawResultsPerModule(), pair.first, FEATURE_COL, QueryConfig.clone(qc) + .setDistanceWeights(pair.second) + .setDistance(ReadableQueryConfig.Distance.manhattan)); + + for (Map row : rows) { + + String segment = row.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + + if (!segmentDistancesMap.containsKey(segment)) { + segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); + } + + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, row.get(PERSON_ID_COL).getInt()), row.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble()); + + } + + ++queryPersonId; + + } + + ArrayList results = new ArrayList<>(segmentDistancesMap.size()); + + //compute assignment + if (queryPersonId == 1) { //only one query skeleton + for (String segment : segmentDistancesMap.keySet()) { + TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); + double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); + results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist))); + } + results.sort(SegmentScoreElement.SCORE_COMPARATOR); + return results.subList(0, Math.min(results.size(), qc.getRawResultsPerModule()) - 1); + } + + + //more than query skeleton + for (String segment : segmentDistancesMap.keySet()) { + + TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); + + if (distances.isEmpty()) { + continue; //should never happen + } + + Set personIds = distances.keySet().stream().map(p -> p.second).collect(Collectors.toSet()); + + if (personIds.size() == 1) { //only one retrieved skeleton + double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); + results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist) / skeletons.size())); + continue; + } + + //more than one retrieved skeletons + + double[][] costs = new double[skeletons.size()][personIds.size()]; + TIntIntHashMap inversePersonIdMapping = new TIntIntHashMap(personIds.size()); + int i = 0; + for (int personId : personIds){ + inversePersonIdMapping.put(personId, i++); + } + + for (Pair p : distances.keySet()) { + costs[p.first][inversePersonIdMapping.get(p.second)] = -distances.get(p); + } + + HungarianAlgorithm hungarianAlgorithm = new HungarianAlgorithm(costs); + int[] assignment = hungarianAlgorithm.execute(); + + double scoreSum = 0; + + for (i = 0; i < Math.min(personIds.size(), skeletons.size()); ++i) { + scoreSum += this.correspondence.applyAsDouble(-costs[i][assignment[i]]); + } + + results.add(new SegmentScoreElement(segment, scoreSum / skeletons.size())); + + } + - return null; + return results; } private Pair getAnglesandWeights(Skeleton skeleton) { From a0b1ac2366247bbf774a67c2097aa5c04ef9faa5 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 24 Jan 2022 17:31:33 +0100 Subject: [PATCH 15/72] Added import mechanism for pre-extracted skeletons --- .../cineast/core/features/SkeletonPose.java | 101 +++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 532c90f0c..b05eecd42 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -1,23 +1,34 @@ package org.vitrivr.cineast.core.features; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import georegression.struct.point.Point2D_F32; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TObjectDoubleHashMap; import org.apache.commons.lang3.NotImplementedException; +import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.score.SegmentScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.PersistentTuple; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.HungarianAlgorithm; +import java.io.File; +import java.io.IOException; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -51,9 +62,39 @@ public void initalizePersistentLayer(Supplier supply) { ); } + protected void persist(String shotId, Collection skeletons) { + if (skeletons == null || skeletons.isEmpty()) { + return; + } + + List tuples = new ArrayList<>(skeletons.size()); + + int i = 0; + for (Skeleton skeleton : skeletons) { + Pair pair = getAnglesandWeights(skeleton); + tuples.add(this.phandler.generateTuple( + shotId, i++, pair.first, pair.second + )); + } + + this.phandler.persist(tuples); + } + + private List detectSkeletons(MultiImage img) { + throw new NotImplementedException("not currently available"); + } + @Override public void processSegment(SegmentContainer segmentContainer) { - throw new NotImplementedException("not currently available"); + + VideoFrame representativeFrame = segmentContainer.getMostRepresentativeFrame(); + + if (representativeFrame == VideoFrame.EMPTY_VIDEO_FRAME) { + return; + } + + persist(segmentContainer.getId(), detectSkeletons(representativeFrame.getImage())); + } @Override @@ -151,7 +192,6 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc } - return results; } @@ -192,4 +232,61 @@ private static float min(float f, float g, float h) { return Math.min(f, Math.min(g, h)); } + public static void main(String[] args) throws IOException { + + File baseDir = new File("../../Downloads/VBS2022/VBS2022"); + File[] folders = baseDir.listFiles(File::isDirectory); + + ObjectMapper mapper = new ObjectMapper(); + TypeReference> typeRef = new TypeReference<>() { }; + + DatabaseConfig config = new DatabaseConfig(); + config.setHost("localhost"); + config.setPort(1865); + + CottontailWrapper ctWrapper = new CottontailWrapper(config, true); + + + SkeletonPose sp = new SkeletonPose(); + sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); + sp.init(() -> new CottontailWriter(ctWrapper), 100); + + + for (File folder : folders) { + for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { + + String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); + + List derialized = mapper.readValue(file, typeRef); + + List skeletons = derialized.stream().map(SkeletonEntry::toSkeleton).collect(Collectors.toList()); + + sp.persist(segmentId, skeletons); + + } + System.out.println("done with " + folder.getName()); + } + + } + + static class SkeletonEntry{ + public int person_id; + public List> pose_keypoints; + + public Skeleton toSkeleton() { + float[] weights = new float[17]; + float[] coordinates = new float[34]; + + for (int i = 0; i < 17; ++i) { + coordinates[2 * i] = pose_keypoints.get(i).get(0); + coordinates[2 * i + 1] = pose_keypoints.get(i).get(1); + weights[i] = pose_keypoints.get(i).get(2); + } + + return new Skeleton(coordinates, weights); + + } + + } + } From 7d557b921d3fc0cc34b39d5ac3e8a8422ff2b002 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 25 Jan 2022 08:30:13 +0100 Subject: [PATCH 16/72] Fix in entity creation for SkeletonPose --- .../java/org/vitrivr/cineast/core/features/SkeletonPose.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index b05eecd42..da2a9d47c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -55,7 +55,6 @@ public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { @Override public void initalizePersistentLayer(Supplier supply) { supply.get().createFeatureEntity(this.tableName, false, - new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) @@ -224,6 +223,10 @@ private Pair getAnglesandWeights(Skeleton skeleton) { } + private void buildQuery() { + //... + } + private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { return (float) (Math.atan2(p2.y - c.y, p2.x - c.x) - Math.atan2(p1.y - c.y, p1.x - c.x)); } From d11ca23becb865293c3678037c83cf53176db774 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 25 Jan 2022 19:26:43 +0100 Subject: [PATCH 17/72] Towards querying with new cottontail functionality --- .../cottontaildb/CottontailEntityCreator.java | 125 ++++---- .../db/cottontaildb/CottontailSelector.java | 204 ++++++------- .../db/cottontaildb/CottontailWrapper.java | 5 +- .../db/cottontaildb/CottontailWriter.java | 11 +- .../cineast/core/features/SkeletonPose.java | 175 ++++++++--- .../CottontailIntegrationTest.java | 2 +- .../cottontaildb/CottontailMetadataTest.java | 4 +- .../cineast/standalone/cli/CineastCli.java | 4 +- .../cli/OptimizeEntitiesCommand.java | 6 +- .../cli/db/LSC21TemporalUpdateCommand.java | 286 +++++++++--------- .../importer/handlers/DataImportHandler.java | 13 +- gradle.properties | 2 +- 12 files changed, 450 insertions(+), 387 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 47c11e144..9dd645f6b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -1,17 +1,7 @@ package org.vitrivr.cineast.core.db.cottontaildb; -import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.BITSET; -import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.TEXT; -import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR; -import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; - import io.grpc.StatusRuntimeException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Objects; -import java.util.Optional; -import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; @@ -19,17 +9,21 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; import org.vitrivr.cottontail.client.language.basics.Type; -import org.vitrivr.cottontail.client.language.ddl.AboutEntity; -import org.vitrivr.cottontail.client.language.ddl.CreateEntity; -import org.vitrivr.cottontail.client.language.ddl.CreateIndex; -import org.vitrivr.cottontail.client.language.ddl.CreateSchema; -import org.vitrivr.cottontail.client.language.ddl.DropEntity; -import org.vitrivr.cottontail.client.language.ddl.ListSchemas; +import org.vitrivr.cottontail.client.language.ddl.*; import org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Objects; +import java.util.Optional; + +import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.*; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + public final class CottontailEntityCreator implements EntityCreator { public static final String COTTONTAIL_PREFIX = "cottontail"; @@ -40,11 +34,6 @@ public final class CottontailEntityCreator implements EntityCreator { */ private final CottontailWrapper cottontail; - public CottontailEntityCreator(DatabaseConfig config) { - this.cottontail = new CottontailWrapper(config, false); - init(); - } - public CottontailEntityCreator(CottontailWrapper cottontailWrapper) { this.cottontail = cottontailWrapper; init(); @@ -56,18 +45,18 @@ public CottontailEntityCreator(CottontailWrapper cottontailWrapper) { private void init() { final long txId = this.cottontail.client.begin(); try { - final ListSchemas list = new ListSchemas(); - final TupleIterator iterator = this.cottontail.client.list(list, txId); + final ListSchemas list = new ListSchemas().txId(txId); + final TupleIterator iterator = this.cottontail.client.list(list); boolean exists = false; while (iterator.hasNext()) { - TupleIterator.Tuple next = iterator.next(); + Tuple next = iterator.next(); if (Objects.equals(next.asString(Constants.COLUMN_NAME_DBO), CottontailWrapper.FQN_CINEAST_SCHEMA)) { exists = true; break; } } if (!exists) { - this.cottontail.client.create(new CreateSchema(CottontailWrapper.CINEAST_SCHEMA), txId); + this.cottontail.client.create(new CreateSchema(CottontailWrapper.CINEAST_SCHEMA).txId(txId)); } this.cottontail.client.commit(txId); } catch (StatusRuntimeException e) { @@ -83,10 +72,11 @@ public boolean createTagEntity() { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + TagReader.TAG_ENTITY_NAME; final CreateEntity create = new CreateEntity(entityName) - .column(TagReader.TAG_ID_COLUMNNAME, Type.STRING, -1, false) - .column(TagReader.TAG_NAME_COLUMNNAME, Type.STRING, -1, false) - .column(TagReader.TAG_DESCRIPTION_COLUMNNAME, Type.STRING, -1, false); - this.cottontail.client.create(create, txId); + .column(TagReader.TAG_ID_COLUMNNAME, Type.STRING, -1, false) + .column(TagReader.TAG_NAME_COLUMNNAME, Type.STRING, -1, false) + .column(TagReader.TAG_DESCRIPTION_COLUMNNAME, Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(create); /* tag ids should be unique */ this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, IndexType.HASH_UQ, txId); @@ -110,11 +100,12 @@ public boolean createMultiMediaObjectsEntity() { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + MediaObjectDescriptor.ENTITY; final CreateEntity entity = new CreateEntity(entityName) - .column(MediaObjectDescriptor.FIELDNAMES[0], Type.STRING, -1, false) - .column(MediaObjectDescriptor.FIELDNAMES[1], Type.INTEGER, -1, false) - .column(MediaObjectDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaObjectDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaObjectDescriptor.FIELDNAMES[0], Type.STRING, -1, false) + .column(MediaObjectDescriptor.FIELDNAMES[1], Type.INTEGER, -1, false) + .column(MediaObjectDescriptor.FIELDNAMES[2], Type.STRING, -1, false) + .column(MediaObjectDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create index. */ this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); @@ -133,14 +124,15 @@ public boolean createSegmentEntity() { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + MediaSegmentDescriptor.ENTITY; final CreateEntity entity = new CreateEntity(entityName) - .column(MediaSegmentDescriptor.FIELDNAMES[0], Type.STRING, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[1], Type.STRING, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[2], Type.INTEGER, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[3], Type.INTEGER, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[4], Type.INTEGER, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[5], Type.DOUBLE, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[6], Type.DOUBLE, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaSegmentDescriptor.FIELDNAMES[0], Type.STRING, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[1], Type.STRING, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[2], Type.INTEGER, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[3], Type.INTEGER, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[4], Type.INTEGER, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[5], Type.DOUBLE, -1, false) + .column(MediaSegmentDescriptor.FIELDNAMES[6], Type.DOUBLE, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create indexes. */ this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); @@ -160,11 +152,12 @@ public boolean createMetadataEntity(String tableName) { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; final CreateEntity entity = new CreateEntity(entityName) - .column(MediaObjectMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) - .column(MediaObjectMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) - .column(MediaObjectMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaObjectMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaObjectMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) + .column(MediaObjectMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) + .column(MediaObjectMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) + .column(MediaObjectMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create Index. */ this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); @@ -183,11 +176,12 @@ public boolean createSegmentMetadataEntity(String tableName) { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; final CreateEntity entity = new CreateEntity(entityName) - .column(MediaSegmentMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) - .column(MediaSegmentMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) - .column(MediaSegmentMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaSegmentMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaSegmentMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) + .column(MediaSegmentMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) + .column(MediaSegmentMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) + .column(MediaSegmentMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create Index. */ this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); @@ -203,8 +197,8 @@ public boolean createSegmentMetadataEntity(String tableName) { @Override public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { final AttributeDefinition[] attributes = Arrays.stream(featureNames) - .map(s -> new AttributeDefinition(s, VECTOR, length)) - .toArray(AttributeDefinition[]::new); + .map(s -> new AttributeDefinition(s, VECTOR, length)) + .toArray(AttributeDefinition[]::new); return this.createFeatureEntity(featureEntityName, unique, attributes); } @@ -228,7 +222,7 @@ public boolean createIdEntity(String entityName, AttributeDefinition... attribut @Override public boolean createEntity(String entityName, AttributeDefinition... attributes) { return this.createEntity( - new org.vitrivr.cineast.core.db.setup.EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build() + new org.vitrivr.cineast.core.db.setup.EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build() ); } @@ -238,16 +232,15 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d try { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(); - final CreateEntity entity = new CreateEntity(entityName); + final CreateEntity entity = new CreateEntity(entityName).txId(txId); for (AttributeDefinition attribute : def.getAttributes()) { int length = -1; if ((attribute.getType() == VECTOR || attribute.getType() == BITSET) && attribute.getLength() > 0) { length = attribute.getLength(); } entity.column(attribute.getName(), mapAttributeType(attribute.getType()), length, false); - } - this.cottontail.client.create(entity, txId); + this.cottontail.client.create(entity); /* Create Index. */ for (AttributeDefinition attribute : def.getAttributes()) { @@ -287,7 +280,7 @@ public boolean createHashNonUniqueIndex(String entityName, String column) { public boolean existsEntity(String entityName) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(entityName)); try { - final TupleIterator results = this.cottontail.client.about(about, null); + final TupleIterator results = this.cottontail.client.about(about); return results.hasNext(); } catch (StatusRuntimeException e) { return false; @@ -299,7 +292,7 @@ public boolean dropEntity(String entityName) { final long txId = this.cottontail.client.begin(); try { final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - this.cottontail.client.drop(new DropEntity(fqn), txId); + this.cottontail.client.drop(new DropEntity(fqn).txId(txId)); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -322,9 +315,13 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { case VECTOR: return Type.FLOAT_VECTOR; case BITSET: - return Type.BOOL_VECTOR; + return Type.BOOLEAN_VECTOR; case FLOAT: return Type.FLOAT; + /*case GEOGRAPHY: + return Type.GEOGRAPHY; + case GEOMETRY: + return Type.GEOMETRY;*/ case INT: return Type.INTEGER; case LONG: @@ -340,7 +337,7 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { private void createIndex(String entityName, String attribute, IndexType type, long txId) { final String indexName = entityName + ".idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute); - this.cottontail.client.create(index, txId); + final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); + this.cottontail.client.create(index); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 5445eab29..2c411b5ab 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -1,24 +1,6 @@ package org.vitrivr.cineast.core.db.cottontaildb; -import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; -import static org.vitrivr.cineast.core.util.CineastConstants.DOMAIN_COL_NAME; -import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import static org.vitrivr.cineast.core.util.CineastConstants.KEY_COL_NAME; - -import com.google.common.collect.Lists; import io.grpc.StatusRuntimeException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; @@ -30,15 +12,23 @@ import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; +import org.vitrivr.cottontail.client.language.basics.Direction; +import org.vitrivr.cottontail.client.language.basics.Distances; import org.vitrivr.cottontail.client.language.ddl.AboutEntity; import org.vitrivr.cottontail.client.language.dql.Query; import org.vitrivr.cottontail.client.language.extensions.And; import org.vitrivr.cottontail.client.language.extensions.Literal; import org.vitrivr.cottontail.client.language.extensions.Or; import org.vitrivr.cottontail.client.language.extensions.Predicate; -import org.vitrivr.cottontail.grpc.CottontailGrpc; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Knn; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.vitrivr.cineast.core.util.CineastConstants.*; public final class CottontailSelector implements DBSelector { @@ -47,7 +37,7 @@ public final class CottontailSelector implements DBSelector { /** * Internal reference to the {@link CottontailWrapper} used by this {@link CottontailSelector}. */ - private final CottontailWrapper cottontail; + public final CottontailWrapper cottontail; /** * The fully qualified name of the entity handled by this {@link CottontailSelector}. @@ -65,18 +55,16 @@ public boolean open(String name) { } @Override - public void close() { - this.cottontail.close(); - } + public void close() { /* No op. */ } /** * if {@link ReadableQueryConfig#getRelevantSegmentIds()} is null, the where-clause will be left empty */ @Override public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config).select(GENERIC_ID_COLUMN_QUALIFIER, DB_DISTANCE_VALUE_QUALIFIER); + final Query query = knn(k, vector, column, config); try { - return handleNearestNeighbourResponse(this.cottontail.client.query(query, null), distanceElementClass); + return handleNearestNeighbourResponse(this.cottontail.client.query(query), distanceElementClass); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getNearestNeighboursGeneric(): {}", e.getMessage()); return new ArrayList<>(0); @@ -90,9 +78,9 @@ public List getBatchedNearestNeighbours(int k, Li @Override public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config).select("*"); + final Query query = knn(k, vector, column, config); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getNearestNeighbourRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -101,12 +89,12 @@ public List> getNearestNeighbourRows(int k, f @Override public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { - final Query query = new Query(this.fqn).select(vectorName).where(new Literal(fieldName, "==", value.toObject())); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())); try { - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); final List _return = new LinkedList<>(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); _return.add(t.asFloatVector(vectorName)); } return _return; @@ -118,9 +106,9 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v @Override public List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName) { - final Query query = new Query(this.fqn).select(vectorName).where(new Literal(fieldName, "==", value.toObject())); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())); try { - return toSingleCol(this.cottontail.client.query(query, null), vectorName); + return toSingleCol(this.cottontail.client.query(query), vectorName); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getFeatureVectorsGeneric(): {}", e.getMessage()); return new ArrayList<>(0); @@ -130,9 +118,9 @@ public List getFeatureVectorsGeneric(String fieldName, Pr @Override public List> getRows(String fieldName, Iterable values) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, "IN", mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, "IN", mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -142,9 +130,9 @@ public List> getRows(String fieldName, Iterab @Override public List> getRows(String fieldName, List values) { final Object[] mapped = values.toArray(); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, "IN", mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, "IN", mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -155,9 +143,9 @@ public List> getRows(String fieldName, List> getRows(String fieldName, RelationalOperator operator, Iterable values) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); final String op = toOperator(operator); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, op, mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, op, mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -167,30 +155,22 @@ public List> getRows(String fieldName, Relati @Override public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { /* Prepare plain query. */ - final Query query = new Query(this.fqn).select( - new kotlin.Pair<>("*", null) - ); + final String predicate = Arrays.stream(terms).map(String::trim).collect(Collectors.joining(" OR ")); + final Query query = new Query(this.fqn) + .select("*", null) + .fulltext(fieldname, predicate, DB_DISTANCE_VALUE_QUALIFIER); /* Process predicates. */ - final List atomics = Arrays.stream(terms).map(t -> new Literal(fieldname, "MATCH", t)).collect(Collectors.toList()); - final Optional predicates = atomics.stream().reduce(Or::new); if (queryConfig != null && !queryConfig.getRelevantSegmentIds().isEmpty()) { final Set relevant = queryConfig.getRelevantSegmentIds(); final Literal segmentIds = new Literal(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); - if (predicates.isPresent()) { - query.where(new And(segmentIds, predicates.get())); - } else { - query.where(segmentIds); - } + query.where(new And(segmentIds, new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0))); } else { - predicates.ifPresent(query::where); + query.where(new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0)); } - Map mappings = new HashMap<>(); - mappings.put("score", DB_DISTANCE_VALUE_QUALIFIER); - try { - return processResults(this.cottontail.client.query(query, null), mappings); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getFulltextRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -202,9 +182,11 @@ public List> getRowsAND(List> getRowsAND(List predicates = atomics.stream().reduce(And::new); if (qc != null && !qc.getRelevantSegmentIds().isEmpty()) { final Set relevant = qc.getRelevantSegmentIds(); @@ -224,8 +208,9 @@ public List> getRowsAND(List(0); @@ -234,25 +219,23 @@ public List> getRowsAND(List> getMetadataBySpec(List spec) { - Query query = new Query(this.fqn); - query.select("*"); - Optional predicates = generateQueryFromMetadataSpec(spec); + final Query query = new Query(this.fqn).select("*", null); + final Optional predicates = generateQueryFromMetadataSpec(spec); predicates.ifPresent(query::where); - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } @Override public List> getMetadataByIdAndSpec(List ids, List spec, String idColName) { - Query query = new Query(this.fqn); - query.select("*"); - Optional predicates = generateQueryFromMetadataSpec(spec); + final Query query = new Query(this.fqn).select("*", null); + final Optional predicates = generateQueryFromMetadataSpec(spec); final Literal segmentIds = new Literal(idColName, "IN", ids.toArray()); if (predicates.isPresent()) { query.where(new And(segmentIds, predicates.get())); } else { query.where(segmentIds); } - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } @@ -268,7 +251,7 @@ public Optional generateQueryFromMetadataSpec(List> reduce = atomics.stream().reduce((res, el) -> { + final Optional> reduce = atomics.stream().reduce((res, el) -> { if (!res.isPresent() && !el.isPresent()) { return Optional.empty(); } @@ -286,9 +269,9 @@ public Optional generateQueryFromMetadataSpec(List getAll(String column) { - final Query query = new Query(this.fqn).select(column); + final Query query = new Query(this.fqn).select(column, null); try { - return toSingleCol(this.cottontail.client.query(query, null), column); + return toSingleCol(this.cottontail.client.query(query), column); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -297,12 +280,15 @@ public List getAll(String column) { @Override public List> getAll(List columns, int limit) { - final Query query = new Query(this.fqn).select(columns.toArray(new String[]{})); + final Query query = new Query(this.fqn); + for (String c : columns) { + query.select(c, null); + } if (limit > 0) { query.limit(limit); } try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -311,9 +297,9 @@ public List> getAll(List columns, int @Override public List getUniqueValues(String column) { - final Query query = new Query(this.fqn).distinct(column); + final Query query = new Query(this.fqn).distinct(column, null); try { - return toSingleCol(this.cottontail.client.query(query, null), column); + return toSingleCol(this.cottontail.client.query(query), column); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getUniqueValues(): {}", e.getMessage()); return new ArrayList<>(0); @@ -321,12 +307,12 @@ public List getUniqueValues(String column) { } public Map countDistinctValues(String column) { - final Query query = new Query(this.fqn).select("*"); + final Query query = new Query(this.fqn).select("*", null); final Map count = new HashMap<>(); try { - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); count.merge(t.asString(column), 1, (old, one) -> old + 1); } return count; @@ -338,9 +324,9 @@ public Map countDistinctValues(String column) { @Override public List> getAll() { - final Query query = new Query(this.fqn).select("*"); + final Query query = new Query(this.fqn).select("*", null); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -350,10 +336,10 @@ public List> getAll() { @Override public boolean existsEntity(String name) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(name)); - try { - final TupleIterator results = this.cottontail.client.about(about, null); + try (final TupleIterator results = this.cottontail.client.about(about)) { return results.hasNext(); - } catch (StatusRuntimeException e) { + } catch (Exception e) { + LOGGER.error("Failed to close TupleIterator!", e); return false; } } @@ -367,10 +353,6 @@ public boolean ping() { } } - List toLiteralList(String s) { - return Lists.newArrayList(CottontailGrpc.Literal.newBuilder().setStringData(s).build()); - } - /** * Creates and returns a basic {@link Query} object for the given kNN parameters. * @@ -381,17 +363,13 @@ List toLiteralList(String s) { * @return {@link Query} */ private Query knn(int k, float[] vector, String column, ReadableQueryConfig config) { - final Optional weights = config.getDistanceWeights(); final Set relevant = config.getRelevantSegmentIds(); - final String distance = toDistance(config.getDistance().orElse(Distance.manhattan)); - final Query query = new Query(this.fqn); - - /* Add weights (optional). */ - if (weights.isPresent()) { - query.knn(column, k, distance, vector, weights.get()); - } else { - query.knn(column, k, distance, vector, null); - } + final Distances distance = toDistance(config.getDistance().orElse(Distance.manhattan)); + final Query query = new Query(this.fqn) + .select(GENERIC_ID_COLUMN_QUALIFIER, null) + .distance(column, vector, distance, DB_DISTANCE_VALUE_QUALIFIER) + .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.ASC) + .limit(k); /* Add relevant segments (optional). */ if (!relevant.isEmpty()) { @@ -404,16 +382,12 @@ private Query knn(int k, float[] vector, String column, ReadableQueryConfig conf private static List> processResults(TupleIterator results, Map mappings) { final List> _return = new LinkedList<>(); final StopWatch watch = StopWatch.createStarted(); - final Collection columns = results.getColumns(); + final Collection columns = results.getSimpleNames(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); final Map map = new HashMap<>(results.getNumberOfColumns()); for (String c : columns) { - if (mappings.containsKey(c)) { - map.put(mappings.get(c), PrimitiveTypeProvider.fromObject(t.get(c))); - } else { - map.put(c, PrimitiveTypeProvider.fromObject(t.get(c))); - } + map.put(mappings.getOrDefault(c, c), PrimitiveTypeProvider.fromObject(t.get(c))); } _return.add(map); } @@ -441,7 +415,7 @@ private static List> processResults(TupleIter private List toSingleCol(TupleIterator results, String colName) { final List _return = new LinkedList<>(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); _return.add(PrimitiveTypeProvider.fromObject(t.get(colName))); } return _return; @@ -458,9 +432,9 @@ private static List handleNearestNeighbourRespons final List result = new LinkedList<>(); while (response.hasNext()) { try { - final TupleIterator.Tuple t = response.next(); - final String id = t.get(GENERIC_ID_COLUMN_QUALIFIER).toString(); /* This should be fine. */ - double distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); + final Tuple t = response.next(); + final String id = t.asString(GENERIC_ID_COLUMN_QUALIFIER); + double distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); /* This should be fine. */ T e = DistanceElement.create(distanceElementClass, id, distance); result.add(e); } catch (NullPointerException e) { @@ -476,23 +450,23 @@ private static List handleNearestNeighbourRespons * @param distance {@link Distance} to convert. * @return {@link String} Name of Cottontail DB distance. */ - private static String toDistance(Distance distance) { + private static Distances toDistance(Distance distance) { switch (distance) { case manhattan: - return Knn.Distance.L1.toString(); + return Distances.L1; case euclidean: - return Knn.Distance.L2.toString(); + return Distances.L2; case squaredeuclidean: - return Knn.Distance.L2SQUARED.toString(); + return Distances.L2SQUARED; case chisquared: - return Knn.Distance.CHISQUARED.toString(); + return Distances.CHISQUARED; case cosine: - return Knn.Distance.COSINE.toString(); + return Distances.COSINE; case haversine: - return Knn.Distance.HAVERSINE.toString(); + return Distances.HAVERSINE; default: LOGGER.error("distance '{}' not supported by cottontail", distance); - throw new IllegalArgumentException("Distance '" + distance.toString() + "' not supported by Cottontail DB."); + throw new IllegalArgumentException("Distance '" + distance + "' not supported by Cottontail DB."); } } @@ -531,7 +505,7 @@ private static String toOperator(RelationalOperator op) { case IN: return "IN"; default: - throw new IllegalArgumentException("Operator '" + op.toString() + "' not supported by Cottontail DB."); + throw new IllegalArgumentException("Operator '" + op + "' not supported by Cottontail DB."); } } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 65a83585d..39e811b5c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -2,12 +2,13 @@ import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; -import org.vitrivr.cottontail.client.stub.SimpleClient; +import org.vitrivr.cottontail.client.SimpleClient; + +import java.util.concurrent.TimeUnit; public final class CottontailWrapper implements AutoCloseable { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index eb5a16812..7727a4ee3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -2,18 +2,19 @@ import io.grpc.StatusRuntimeException; -import java.util.List; import org.apache.logging.log4j.LogManager; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; import org.vitrivr.cottontail.client.language.dml.BatchInsert; import org.vitrivr.cottontail.client.language.dml.Insert; import org.vitrivr.cottontail.client.language.dql.Query; import org.vitrivr.cottontail.client.language.extensions.Literal; +import java.util.List; + public final class CottontailWriter extends AbstractPersistencyWriter { /** @@ -46,7 +47,7 @@ public void close() { @Override public boolean exists(String key, String value) { final Query query = new Query(this.fqn).exists().where(new Literal(key, "=", value)); - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); final Boolean b = results.next().asBoolean("exists"); if (b != null) { return b; @@ -74,13 +75,13 @@ public boolean persist(List tuples) { insert.append(values); if (insert.size() >= Constants.MAX_PAGE_SIZE_BYTES) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); - this.cottontail.client.insert(insert, txId); + this.cottontail.client.insert(insert); insert = new BatchInsert().into(this.fqn).columns(this.names); } } if (insert.getBuilder().getInsertsCount() > 0) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); - this.cottontail.client.insert(insert, txId); + this.cottontail.client.insert(insert); } this.cottontail.client.commit(txId); long stop = System.currentTimeMillis(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index da2a9d47c..55f8c5531 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -9,10 +9,10 @@ import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.FloatArrayIterable; import org.vitrivr.cineast.core.data.Pair; import org.vitrivr.cineast.core.data.Skeleton; import org.vitrivr.cineast.core.data.frames.VideoFrame; -import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.score.SegmentScoreElement; @@ -20,12 +20,17 @@ import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.HungarianAlgorithm; +import org.vitrivr.cottontail.client.SimpleClient; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; +import org.vitrivr.cottontail.grpc.CottontailGrpc; import java.io.File; import java.io.IOException; @@ -58,7 +63,7 @@ public void initalizePersistentLayer(Supplier supply) { new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) - ); + ); } protected void persist(String shotId, Collection skeletons) { @@ -105,6 +110,12 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc return Collections.emptyList(); } + if (!(this.selector instanceof CottontailSelector)) { + return Collections.emptyList(); + } + + SimpleClient client = ((CottontailSelector) this.selector).cottontail.client; + HashMap>> segmentDistancesMap = new HashMap<>(qc.getRawResultsPerModule() * skeletons.size()); int queryPersonId = 0; @@ -114,19 +125,26 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc Pair pair = getAnglesandWeights(skeleton); - List> rows = this.selector.getNearestNeighbourRows(qc.getRawResultsPerModule(), pair.first, FEATURE_COL, QueryConfig.clone(qc) - .setDistanceWeights(pair.second) - .setDistance(ReadableQueryConfig.Distance.manhattan)); +// List> rows = this.selector.getNearestNeighbourRows(qc.getRawResultsPerModule(), pair.first, FEATURE_COL, QueryConfig.clone(qc) +// .setDistanceWeights(pair.second) +// .setDistance(ReadableQueryConfig.Distance.manhattan)); + + + TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); + + + + while (tuples.hasNext()) { - for (Map row : rows) { + Tuple tuple = tuples.next(); - String segment = row.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + String segment = tuple.asString(GENERIC_ID_COLUMN_QUALIFIER); if (!segmentDistancesMap.containsKey(segment)) { segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); } - segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, row.get(PERSON_ID_COL).getInt()), row.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble()); + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble(DB_DISTANCE_VALUE_QUALIFIER)); } @@ -170,7 +188,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc double[][] costs = new double[skeletons.size()][personIds.size()]; TIntIntHashMap inversePersonIdMapping = new TIntIntHashMap(personIds.size()); int i = 0; - for (int personId : personIds){ + for (int personId : personIds) { inversePersonIdMapping.put(personId, i++); } @@ -196,35 +214,62 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc private Pair getAnglesandWeights(Skeleton skeleton) { float[] angles = new float[]{ - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ANKLE),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ANKLE)), - - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_WRIST),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW),skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER),skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_WRIST)) + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ANKLE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ANKLE)), + + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_WRIST), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW)), + angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_WRIST)) }; float[] weights = new float[]{ - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ANKLE),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ANKLE)), - - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_WRIST),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW),skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER),skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_WRIST)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ANKLE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ANKLE)), + + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_WRIST), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW)), + min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_WRIST)), }; return new Pair<>(angles, weights); } - private void buildQuery() { - //... + private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { + return CottontailGrpc.QueryMessage.newBuilder().setQuery( + CottontailGrpc.Query.newBuilder().setFrom( + CottontailGrpc.From.newBuilder().setScan(CottontailGrpc.Scan.newBuilder().setEntity(CottontailGrpc.EntityName.newBuilder().setName(this.tableName).setSchema(CottontailGrpc.SchemaName.newBuilder().setName("cineast")))) + ).setProjection( + CottontailGrpc.Projection.newBuilder() + .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) + .addElements( + CottontailGrpc.Projection.ProjectionElement.newBuilder().setFunction(/* Distance function */ + CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw")).addArguments( + CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(FEATURE_COL)) + ).addArguments( + CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) + ))) + ).addArguments( + CottontailGrpc.Expression.newBuilder().setFunction(/* Nested, min() function */ + CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("min")).addArguments( + CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) + ).addArguments( + CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) + ))) + ) + ) + ) + ) + ) + ).setLimit(limit).build()).build(); } private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { @@ -241,7 +286,8 @@ public static void main(String[] args) throws IOException { File[] folders = baseDir.listFiles(File::isDirectory); ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference<>() { }; + TypeReference> typeRef = new TypeReference<>() { + }; DatabaseConfig config = new DatabaseConfig(); config.setHost("localhost"); @@ -251,28 +297,77 @@ public static void main(String[] args) throws IOException { SkeletonPose sp = new SkeletonPose(); - sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); - sp.init(() -> new CottontailWriter(ctWrapper), 100); - for (File folder : folders) { - for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { + boolean insert = false; + + if (insert) { + sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); + sp.init(() -> new CottontailWriter(ctWrapper), 100); + + + for (File folder : folders) { + for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { + + String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); + + List derialized = mapper.readValue(file, typeRef); + + List skeletons = derialized.stream().map(SkeletonEntry::toSkeleton).collect(Collectors.toList()); + + sp.persist(segmentId, skeletons); + + } + System.out.println("done with " + folder.getName()); + } + } + + + sp.init(() -> new CottontailSelector(ctWrapper)); + + Skeleton skeleton = mapper.readValue(new File(baseDir, "00001/shot00001_10_RKF.json"), typeRef).get(0).toSkeleton(); + + SegmentContainer container = new SegmentContainer() { + @Override + public String getId() { + return null; + } + + @Override + public String getSuperId() { + return null; + } - String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); + @Override + public void setId(String id) { - List derialized = mapper.readValue(file, typeRef); + } - List skeletons = derialized.stream().map(SkeletonEntry::toSkeleton).collect(Collectors.toList()); + @Override + public void setSuperId(String id) { - sp.persist(segmentId, skeletons); + } + + @Override + public List getSkeletons() { + return Collections.singletonList(skeleton); + } + }; + + List results = sp.getSimilar(container, new QueryConfig(null).setResultsPerModule(100)); + + int i = 0; + for (ScoreElement element : results) { + System.out.println(i++ + ": " + element); + if (i > 10) { + break; } - System.out.println("done with " + folder.getName()); } } - static class SkeletonEntry{ + static class SkeletonEntry { public int person_id; public List> pose_keypoints; diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java index b85ed73be..212027b1a 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java @@ -19,7 +19,7 @@ public CottontailIntegrationTest() { public void finishSetup() { final CottontailWrapper wrapper = _provider.getWrapper(); final String fqn = wrapper.fqnInput(this.getTestTextTableName()); - wrapper.client.optimize(new OptimizeEntity(fqn), null); + wrapper.client.optimize(new OptimizeEntity(fqn)); wrapper.close(); } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java index 196bc2656..905bc0812 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java @@ -20,8 +20,8 @@ public void finishInitialSetup() { final CottontailWrapper wrapper = _provider.getWrapper(); final String obj = wrapper.fqnInput(this.getTestObjMetaTableName()); final String seg = wrapper.fqnInput(this.getTestSegMetaTableName()); - wrapper.client.optimize(new OptimizeEntity(obj), null); - wrapper.client.optimize(new OptimizeEntity(seg), null); + wrapper.client.optimize(new OptimizeEntity(obj)); + wrapper.client.optimize(new OptimizeEntity(seg)); wrapper.close(); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java index c518a4279..be607a015 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java @@ -3,14 +3,14 @@ import com.github.rvesse.airline.annotations.Cli; import com.github.rvesse.airline.help.Help; import org.vitrivr.cineast.standalone.cli.db.DropTableCommand; -import org.vitrivr.cineast.standalone.cli.db.LSC21TemporalUpdateCommand; +//import org.vitrivr.cineast.standalone.cli.db.LSC21TemporalUpdateCommand; @Cli(name = "cineast-api", description = "The CLI provided by the Cineast API.", commands = { DropTableCommand.class, TagRetrievalCommand.class, OptimizeEntitiesCommand.class, CodebookCommand.class, DatabaseSetupCommand.class, ExtractionCommand.class, ImportCommand.class, ThreeDeeTestCommand.class, RetrieveCommand.class, Help.class, SingleObjRetrievalCommand.class, TextRetrievalCommand.class, DistinctColumnApiCommand.class, - LSC21TemporalUpdateCommand.class}, defaultCommand = Help.class) + /*LSC21TemporalUpdateCommand.class*/}, defaultCommand = Help.class) public class CineastCli { } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java index 975f5324b..5b03b152d 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java @@ -5,8 +5,6 @@ import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cottontail.client.language.ddl.ListEntities; -import org.vitrivr.cottontail.client.language.ddl.OptimizeEntity; @Command(name = "optimize", description = "Optimize all entities for the Cineast schema. This command is only compatible with the Cottontail DB database.") public class OptimizeEntitiesCommand implements Runnable { @@ -22,13 +20,13 @@ public static void optimizeAllCottontailEntities() { return; } try (final CottontailWrapper wrapper = new CottontailWrapper(Config.sharedConfig().getDatabase(), false)) { - System.out.println("Optimizing all entities for schema '" + CottontailWrapper.CINEAST_SCHEMA + "' in Cottontail"); + /* System.out.println("Optimizing all entities for schema '" + CottontailWrapper.CINEAST_SCHEMA + "' in Cottontail"); wrapper.client.list(new ListEntities(CottontailWrapper.CINEAST_SCHEMA), null).forEachRemaining(entity -> { System.out.println("Optimizing entity " + entity); final String name = entity.asString("dbo").replace("warren.", ""); wrapper.client.optimize(new OptimizeEntity(name), null); }); - System.out.println("Finished optimizing all entities"); + System.out.println("Finished optimizing all entities");*/ } } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java index 6d06909f8..9c3cd9773 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java @@ -1,143 +1,143 @@ -package org.vitrivr.cineast.standalone.cli.db; - -import com.github.rvesse.airline.annotations.Command; -import com.github.rvesse.airline.annotations.Option; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import kotlin.Pair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DatabaseConfig.Selector; -import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; -import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; -import org.vitrivr.cineast.core.util.CineastConstants; -import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cineast.standalone.importer.lsc2020.LSCUtilities; -import org.vitrivr.cottontail.client.TupleIterator; -import org.vitrivr.cottontail.client.TupleIterator.Tuple; -import org.vitrivr.cottontail.client.language.dml.Update; -import org.vitrivr.cottontail.client.language.dql.Query; -import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal.Builder; -import org.vitrivr.cottontail.grpc.CottontailGrpc.UpdateMessage.UpdateElement; - -@Command(name = "lsc21-time-update", description = "Updates media segments based on their ID") -public class LSC21TemporalUpdateCommand implements Runnable { - - private final static Logger LOGGER = LogManager.getLogger(LSC21TemporalUpdateCommand.class); - - private static final String ENTITY_NAME = "cineast.cineast_segment"; - - @Option(name = {"-p", - "--progress"}, title = "Progress", description = "Flag to indicate that some progress information is desired") - private boolean progress = false; - - private static MediaSegmentDescriptor convert(Tuple segmentTuple) { - final String oid = (String) segmentTuple.get(CineastConstants.OBJECT_ID_COLUMN_QUALIFIER); - final String sid = (String) segmentTuple.get(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER); - final int number = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_NO_COL_NAME); - final int start = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_START_COL_NAME); - final int end = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_END_COL_NAME); - final double startabs = (Double) segmentTuple.get( - MediaSegmentDescriptor.SEGMENT_STARTABS_COL_NAME); - final double endabs = (Double) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME); - MediaSegmentDescriptor segment = new MediaSegmentDescriptor( - oid, sid, number, start, end, (float) startabs, (float) endabs, true - ); - return segment; - } - - private static List convert(MediaSegmentDescriptor segment) { - return Arrays.stream(MediaSegmentDescriptor.FIELDNAMES).map(name -> UpdateElement.newBuilder() - .setColumn(ColumnName.newBuilder().setName(name).build()) - .setValue(forValue(segment, name)) - .build()).collect(Collectors.toList()); - } - - - private static Literal forValue(MediaSegmentDescriptor segment, String name) { - final Builder builder = Literal.newBuilder(); - - switch (name) { - case CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER: - builder.setStringData(segment.getSegmentId()); - break; - case CineastConstants.OBJECT_ID_COLUMN_QUALIFIER: - builder.setStringData(segment.getObjectId()); - break; - case MediaSegmentDescriptor.SEGMENT_NO_COL_NAME: - builder.setIntData(segment.getSequenceNumber()); - break; - case MediaSegmentDescriptor.SEGMENT_START_COL_NAME: - builder.setIntData(segment.getStart()); - break; - case MediaSegmentDescriptor.SEGMENT_END_COL_NAME: - builder.setIntData(segment.getEnd()); - break; - case MediaSegmentDescriptor.SEGMENT_STARTABS_COL_NAME: - builder.setFloatData(segment.getStartabs()); - break; - case MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME: - builder.setFloatData(segment.getEndabs()); - break; - default: - LOGGER.warn("Cannot parse column" + name + " for segment " + segment.toString()); - } - - return builder.build(); - } - - @Override - @SuppressWarnings("unchecked") - public void run() { - if (Config.sharedConfig().getDatabase().getSelector() != Selector.COTTONTAIL || - Config.sharedConfig().getDatabase().getWriter() != Writer.COTTONTAIL - ) { - System.out.println("Other DB than Cottontail DB not supported (yet). Aborting"); - return; - } - final CottontailWrapper cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); - long txId = cottontail.client.begin(); - long uTxId = cottontail.client.begin(); - final Query query = new Query(ENTITY_NAME).select("*"); - final TupleIterator ti = cottontail.client.query(query, txId); - final List updateElements = new ArrayList<>(); - int counter = 0; - int totalCounter = 0; - while (ti.hasNext()) { - final Tuple t = ti.next(); - final MediaSegmentDescriptor segment = convert(t); - final Optional minuteIdOpt = LSCUtilities.filenameToMinuteId(segment.getSegmentId().substring(4)); - if (!minuteIdOpt.isPresent()) { - LOGGER.warn("Could not update " + segment.getSegmentId()); - continue; - } - final LocalDateTime ldt = LSCUtilities.fromMinuteId(minuteIdOpt.get()); - final long msAbs = ldt.toInstant(ZoneOffset.UTC).toEpochMilli(); - final long msAbsNext = msAbs + 1; - final Update update = new Update(ENTITY_NAME) - .values( - new Pair<>(MediaSegmentDescriptor.SEGMENT_START_COL_NAME, (double) msAbs), - new Pair<>(MediaSegmentDescriptor.SEGMENT_END_COL_NAME, (double) msAbsNext) - ) - .where(new org.vitrivr.cottontail.client.language.extensions.Literal(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())); - cottontail.client.update(update, txId); - totalCounter++; - if (counter++ > 99) { - if (progress) { - System.out.println("Updated " + totalCounter + " rows."); - } - counter = 0; - } - } - cottontail.client.commit(txId); - System.out.println("Done."); - } -} +//package org.vitrivr.cineast.standalone.cli.db; +// +//import com.github.rvesse.airline.annotations.Command; +//import com.github.rvesse.airline.annotations.Option; +//import java.time.LocalDateTime; +//import java.time.ZoneOffset; +//import java.util.ArrayList; +//import java.util.Arrays; +//import java.util.List; +//import java.util.Optional; +//import java.util.stream.Collectors; +//import kotlin.Pair; +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; +//import org.vitrivr.cineast.core.config.DatabaseConfig.Selector; +//import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; +//import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; +//import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; +//import org.vitrivr.cineast.core.util.CineastConstants; +//import org.vitrivr.cineast.standalone.config.Config; +//import org.vitrivr.cineast.standalone.importer.lsc2020.LSCUtilities; +//import org.vitrivr.cottontail.client.TupleIterator; +//import org.vitrivr.cottontail.client.TupleIterator.Tuple; +//import org.vitrivr.cottontail.client.language.dml.Update; +//import org.vitrivr.cottontail.client.language.dql.Query; +//import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; +//import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; +//import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal.Builder; +//import org.vitrivr.cottontail.grpc.CottontailGrpc.UpdateMessage.UpdateElement; +// +//@Command(name = "lsc21-time-update", description = "Updates media segments based on their ID") +//public class LSC21TemporalUpdateCommand implements Runnable { +// +// private final static Logger LOGGER = LogManager.getLogger(LSC21TemporalUpdateCommand.class); +// +// private static final String ENTITY_NAME = "cineast.cineast_segment"; +// +// @Option(name = {"-p", +// "--progress"}, title = "Progress", description = "Flag to indicate that some progress information is desired") +// private boolean progress = false; +// +// private static MediaSegmentDescriptor convert(Tuple segmentTuple) { +// final String oid = (String) segmentTuple.get(CineastConstants.OBJECT_ID_COLUMN_QUALIFIER); +// final String sid = (String) segmentTuple.get(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER); +// final int number = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_NO_COL_NAME); +// final int start = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_START_COL_NAME); +// final int end = (Integer) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_END_COL_NAME); +// final double startabs = (Double) segmentTuple.get( +// MediaSegmentDescriptor.SEGMENT_STARTABS_COL_NAME); +// final double endabs = (Double) segmentTuple.get(MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME); +// MediaSegmentDescriptor segment = new MediaSegmentDescriptor( +// oid, sid, number, start, end, (float) startabs, (float) endabs, true +// ); +// return segment; +// } +// +// private static List convert(MediaSegmentDescriptor segment) { +// return Arrays.stream(MediaSegmentDescriptor.FIELDNAMES).map(name -> UpdateElement.newBuilder() +// .setColumn(ColumnName.newBuilder().setName(name).build()) +// .setValue(forValue(segment, name)) +// .build()).collect(Collectors.toList()); +// } +// +// +// private static Literal forValue(MediaSegmentDescriptor segment, String name) { +// final Builder builder = Literal.newBuilder(); +// +// switch (name) { +// case CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER: +// builder.setStringData(segment.getSegmentId()); +// break; +// case CineastConstants.OBJECT_ID_COLUMN_QUALIFIER: +// builder.setStringData(segment.getObjectId()); +// break; +// case MediaSegmentDescriptor.SEGMENT_NO_COL_NAME: +// builder.setIntData(segment.getSequenceNumber()); +// break; +// case MediaSegmentDescriptor.SEGMENT_START_COL_NAME: +// builder.setIntData(segment.getStart()); +// break; +// case MediaSegmentDescriptor.SEGMENT_END_COL_NAME: +// builder.setIntData(segment.getEnd()); +// break; +// case MediaSegmentDescriptor.SEGMENT_STARTABS_COL_NAME: +// builder.setFloatData(segment.getStartabs()); +// break; +// case MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME: +// builder.setFloatData(segment.getEndabs()); +// break; +// default: +// LOGGER.warn("Cannot parse column" + name + " for segment " + segment.toString()); +// } +// +// return builder.build(); +// } +// +// @Override +// @SuppressWarnings("unchecked") +// public void run() { +// if (Config.sharedConfig().getDatabase().getSelector() != Selector.COTTONTAIL || +// Config.sharedConfig().getDatabase().getWriter() != Writer.COTTONTAIL +// ) { +// System.out.println("Other DB than Cottontail DB not supported (yet). Aborting"); +// return; +// } +// final CottontailWrapper cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); +// long txId = cottontail.client.begin(); +// long uTxId = cottontail.client.begin(); +// final Query query = new Query(ENTITY_NAME).select("*"); +// final TupleIterator ti = cottontail.client.query(query, txId); +// final List updateElements = new ArrayList<>(); +// int counter = 0; +// int totalCounter = 0; +// while (ti.hasNext()) { +// final Tuple t = ti.next(); +// final MediaSegmentDescriptor segment = convert(t); +// final Optional minuteIdOpt = LSCUtilities.filenameToMinuteId(segment.getSegmentId().substring(4)); +// if (!minuteIdOpt.isPresent()) { +// LOGGER.warn("Could not update " + segment.getSegmentId()); +// continue; +// } +// final LocalDateTime ldt = LSCUtilities.fromMinuteId(minuteIdOpt.get()); +// final long msAbs = ldt.toInstant(ZoneOffset.UTC).toEpochMilli(); +// final long msAbsNext = msAbs + 1; +// final Update update = new Update(ENTITY_NAME) +// .values( +// new Pair<>(MediaSegmentDescriptor.SEGMENT_START_COL_NAME, (double) msAbs), +// new Pair<>(MediaSegmentDescriptor.SEGMENT_END_COL_NAME, (double) msAbsNext) +// ) +// .where(new org.vitrivr.cottontail.client.language.extensions.Literal(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())); +// cottontail.client.update(update, txId); +// totalCounter++; +// if (counter++ > 99) { +// if (progress) { +// System.out.println("Updated " + totalCounter + " rows."); +// } +// counter = 0; +// } +// } +// cottontail.client.commit(txId); +// System.out.println("Done."); +// } +//} diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java index df3a1f6ca..c8415ea99 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java @@ -1,12 +1,5 @@ package org.vitrivr.cineast.standalone.importer.handlers; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.DatabaseConfig; @@ -21,6 +14,10 @@ import org.vitrivr.cineast.standalone.monitoring.ImportTaskMonitor; import org.vitrivr.cottontail.client.language.ddl.CreateEntity; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.concurrent.*; + public abstract class DataImportHandler { @@ -51,7 +48,7 @@ protected static void cleanOnDemand(String entityName, String taskName) { DatabaseSetupCommand setupCmd = new DatabaseSetupCommand(); setupCmd.doSetup(); } else { - cottontail.client.create(createEntity, null); + cottontail.client.create(createEntity); LOGGER.info("Re-created entity: {}", createEntity.getBuilder().getDefinition().getEntity().getName()); } } diff --git a/gradle.properties b/gradle.properties index 5a51dfff7..a75991012 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ version_commonstext=1.9 version_commonsio=2.8.0 version_commonslang3=3.12.0 version_commonsmath3=3.6.1 -version_cottontail_proto=0.12.5 +version_cottontail_proto=0.13.0 version_ffmpeg=4.1-1.4.4 version_gson=2.8.6 version_guava=30.1.1-jre From 946381b658edb6110bff2a1e16740a9df145eb27 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 26 Jan 2022 08:53:37 +0100 Subject: [PATCH 18/72] Fixed some issues in query construction, successfully retrieved results --- .../org/vitrivr/cineast/core/features/SkeletonPose.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 55f8c5531..500b0da1a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -38,7 +38,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; public class SkeletonPose extends AbstractFeatureModule { @@ -144,7 +143,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); } - segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble(DB_DISTANCE_VALUE_QUALIFIER)); + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble("manhattanw")); } @@ -248,6 +247,7 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i ).setProjection( CottontailGrpc.Projection.newBuilder() .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) + .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(PERSON_ID_COL))) .addElements( CottontailGrpc.Projection.ProjectionElement.newBuilder().setFunction(/* Distance function */ CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw")).addArguments( @@ -258,7 +258,7 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i ))) ).addArguments( CottontailGrpc.Expression.newBuilder().setFunction(/* Nested, min() function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("min")).addArguments( + CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) ).addArguments( CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( From 8b2d4c0e659eb34b5e6bd9cc847daed63d531329 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 26 Jan 2022 11:01:42 +0100 Subject: [PATCH 19/72] Added SkeletonQueryTermContainer --- .../api/messages/query/QueryTermType.java | 20 +++-------- .../SkeletonQueryTermContainer.java | 33 +++++++++++++++++++ .../cineast/core/features/SkeletonPose.java | 5 +-- 3 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java index f9e891fc6..94ac7f820 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java @@ -1,23 +1,12 @@ package org.vitrivr.cineast.api.messages.query; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.query.containers.*; + import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Optional; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.AudioQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.BooleanQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.IdQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.ImageQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.InstantQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.LocationQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.MotionQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.ParameterisedLocationQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.SemanticMapQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.TagQueryTermContainer; -import org.vitrivr.cineast.core.data.query.containers.TextQueryTermContainer; /** * A {@link QueryTermType} represents the types of query terms used. @@ -34,6 +23,7 @@ public enum QueryTermType { TEXT(TextQueryTermContainer.class), TAG(TagQueryTermContainer.class), SEMANTIC(SemanticMapQueryTermContainer.class), + SKELETON(SkeletonQueryTermContainer.class), /** * Denotes a {@link QueryTerm} containing an Id for a 'More-Like-This' query. This is used over the @link {@link MoreLikeThisQuery} in REST calls. diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java new file mode 100644 index 000000000..db14d4684 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java @@ -0,0 +1,33 @@ +package org.vitrivr.cineast.core.data.query.containers; + +import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; +import org.vitrivr.cineast.core.util.web.DataURLParser; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +public class SkeletonQueryTermContainer extends AbstractQueryTermContainer{ + + private final List skeletons = new ArrayList<>(); + + public SkeletonQueryTermContainer(String data) { + final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); + final String converted = DataURLParser.dataURLtoString(data, "application/json"); + final Skeleton[] skeletons = jsonProvider.toObject(converted, Skeleton[].class); + if (skeletons != null) { + this.skeletons.addAll(Arrays.asList(skeletons)); + } + } + + public SkeletonQueryTermContainer(Collection skeletons) { + this.skeletons.addAll(skeletons); + } + + @Override + public List getSkeletons() { + return this.skeletons; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 500b0da1a..90bc659f7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -325,7 +325,7 @@ public static void main(String[] args) throws IOException { sp.init(() -> new CottontailSelector(ctWrapper)); - Skeleton skeleton = mapper.readValue(new File(baseDir, "00001/shot00001_10_RKF.json"), typeRef).get(0).toSkeleton(); + Skeleton skeleton = mapper.readValue(new File(baseDir, "00006/shot00006_22_RKF.json"), typeRef).get(0).toSkeleton(); SegmentContainer container = new SegmentContainer() { @Override @@ -360,9 +360,6 @@ public List getSkeletons() { for (ScoreElement element : results) { System.out.println(i++ + ": " + element); - if (i > 10) { - break; - } } } From 38fa477a2fae6dbb40dd6adc8dda78b2e805befd Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 27 Jan 2022 11:43:56 +0100 Subject: [PATCH 20/72] Corrected query. --- .../cineast/core/features/SkeletonPose.java | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 90bc659f7..ffcf18452 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -37,7 +37,25 @@ import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; - +import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; +import org.vitrivr.cottontail.grpc.CottontailGrpc.EntityName; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Expression; +import org.vitrivr.cottontail.grpc.CottontailGrpc.FloatVector; +import org.vitrivr.cottontail.grpc.CottontailGrpc.From; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Function; +import org.vitrivr.cottontail.grpc.CottontailGrpc.FunctionName; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Component; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Direction; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Query; +import org.vitrivr.cottontail.grpc.CottontailGrpc.QueryMessage; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Scan; +import org.vitrivr.cottontail.grpc.CottontailGrpc.SchemaName; + +import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; public class SkeletonPose extends AbstractFeatureModule { @@ -132,9 +150,9 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); - + int i = 0; while (tuples.hasNext()) { - + i++; Tuple tuple = tuples.next(); String segment = tuple.asString(GENERIC_ID_COLUMN_QUALIFIER); @@ -143,7 +161,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); } - segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble("manhattanw")); + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble(DB_DISTANCE_VALUE_QUALIFIER)); } @@ -241,34 +259,38 @@ private Pair getAnglesandWeights(Skeleton skeleton) { } private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { - return CottontailGrpc.QueryMessage.newBuilder().setQuery( - CottontailGrpc.Query.newBuilder().setFrom( - CottontailGrpc.From.newBuilder().setScan(CottontailGrpc.Scan.newBuilder().setEntity(CottontailGrpc.EntityName.newBuilder().setName(this.tableName).setSchema(CottontailGrpc.SchemaName.newBuilder().setName("cineast")))) + return QueryMessage.newBuilder().setQuery( + Query.newBuilder().setFrom( + From.newBuilder().setScan(Scan.newBuilder().setEntity(EntityName.newBuilder().setName(this.tableName).setSchema(SchemaName.newBuilder().setName("cineast")))) ).setProjection( - CottontailGrpc.Projection.newBuilder() - .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) - .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(PERSON_ID_COL))) + Projection.newBuilder() + .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) + .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(PERSON_ID_COL))) .addElements( - CottontailGrpc.Projection.ProjectionElement.newBuilder().setFunction(/* Distance function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw")).addArguments( - CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(FEATURE_COL)) + ProjectionElement.newBuilder().setFunction(/* Distance function */ + Function.newBuilder().setName(FunctionName.newBuilder().setName("manhattanw")).addArguments( + Expression.newBuilder().setColumn(ColumnName.newBuilder().setName(FEATURE_COL)) ).addArguments( - CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) + Expression.newBuilder().setLiteral(Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) ))) ).addArguments( - CottontailGrpc.Expression.newBuilder().setFunction(/* Nested, min() function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( - CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) + Expression.newBuilder().setFunction(/* Nested, min() function */ + Function.newBuilder().setName(FunctionName.newBuilder().setName("vmin")).addArguments( + Expression.newBuilder().setColumn(ColumnName.newBuilder().setName(WEIGHT_COL)) ).addArguments( - CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) + Expression.newBuilder().setLiteral(Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) ))) ) ) ) - ) + ).setAlias(ColumnName.newBuilder().setName(DB_DISTANCE_VALUE_QUALIFIER)) ) + ).setOrder( + Order.newBuilder() + .addComponents(Component.newBuilder().setColumn(ColumnName.newBuilder().setName(DB_DISTANCE_VALUE_QUALIFIER)) + .setDirection(Direction.ASCENDING).build()).build() ).setLimit(limit).build()).build(); } @@ -282,7 +304,7 @@ private static float min(float f, float g, float h) { public static void main(String[] args) throws IOException { - File baseDir = new File("../../Downloads/VBS2022/VBS2022"); + File baseDir = new File("/Users/rgasser/Downloads/VBS2022"); File[] folders = baseDir.listFiles(File::isDirectory); ObjectMapper mapper = new ObjectMapper(); @@ -300,7 +322,6 @@ public static void main(String[] args) throws IOException { boolean insert = false; - if (insert) { sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); sp.init(() -> new CottontailWriter(ctWrapper), 100); From 090dfd9367f7104f8cf698ab83dfe79231e1db33 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 1 Feb 2022 20:28:24 +0100 Subject: [PATCH 21/72] Extended function to be evaluated by cottontail --- .../cineast/core/features/SkeletonPose.java | 81 ++++++++++++++----- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 90bc659f7..b7c951060 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -31,6 +31,9 @@ import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.grpc.CottontailGrpc; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Expression; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Function; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; import java.io.File; import java.io.IOException; @@ -132,7 +135,6 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); - while (tuples.hasNext()) { Tuple tuple = tuples.next(); @@ -241,33 +243,70 @@ private Pair getAnglesandWeights(Skeleton skeleton) { } private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { + + float queryWeightSum = 0f; + + for (float w : weights) { + queryWeightSum += w; + } + + Expression vectorDifference = Expression.newBuilder().setFunction(/* Nested, min() function */ + CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( + Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) + ).addArguments( + Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) + ))) + ) + ).build(); + + Expression correctionTerm = Expression.newBuilder().setFunction( + Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("mul") + ).addArguments( //constant + Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData((float) Math.PI)) + ).addArguments( //sub-expression + Expression.newBuilder().setFunction( + Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") + ).addArguments( + Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData(queryWeightSum)) + ).addArguments( + Expression.newBuilder().setFunction(CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vsum")).addArguments(vectorDifference)) + ) + ) + ) + + ).build(); + + + ProjectionElement distanceFunction = ProjectionElement.newBuilder().setFunction(/* Distance function */ + + Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("add")).addArguments( + Expression.newBuilder().setFunction(Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw") + ).addArguments( + Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(FEATURE_COL)) + ).addArguments( + Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) + ))) + ).addArguments( + vectorDifference + )) + ).addArguments( + correctionTerm + ) + ).build(); + + return CottontailGrpc.QueryMessage.newBuilder().setQuery( CottontailGrpc.Query.newBuilder().setFrom( - CottontailGrpc.From.newBuilder().setScan(CottontailGrpc.Scan.newBuilder().setEntity(CottontailGrpc.EntityName.newBuilder().setName(this.tableName).setSchema(CottontailGrpc.SchemaName.newBuilder().setName("cineast")))) + CottontailGrpc.From.newBuilder().setScan(CottontailGrpc.Scan.newBuilder().setEntity(CottontailGrpc.EntityName.newBuilder() + .setName(this.tableName).setSchema(CottontailGrpc.SchemaName.newBuilder().setName("cineast")))) ).setProjection( CottontailGrpc.Projection.newBuilder() .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) .addElements(CottontailGrpc.Projection.ProjectionElement.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(PERSON_ID_COL))) .addElements( - CottontailGrpc.Projection.ProjectionElement.newBuilder().setFunction(/* Distance function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw")).addArguments( - CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(FEATURE_COL)) - ).addArguments( - CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) - ))) - ).addArguments( - CottontailGrpc.Expression.newBuilder().setFunction(/* Nested, min() function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( - CottontailGrpc.Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) - ).addArguments( - CottontailGrpc.Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) - ))) - ) - ) - ) - ) + distanceFunction ) ).setLimit(limit).build()).build(); } From 36acbb77e89829233b382b5fc3e0e1fc73875278 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 2 Feb 2022 10:56:18 +0100 Subject: [PATCH 22/72] Added missing alias for distance column. --- .../cineast/core/features/SkeletonPose.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 18a76cd56..1061baac4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -164,7 +164,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); } - segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asDouble(DB_DISTANCE_VALUE_QUALIFIER)); + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asFloat(DB_DISTANCE_VALUE_QUALIFIER)); } @@ -299,13 +299,13 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i ProjectionElement distanceFunction = ProjectionElement.newBuilder().setFunction(/* Distance function */ - Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("add")).addArguments( - Expression.newBuilder().setFunction(Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("manhattanw") + Function.newBuilder().setName(FunctionName.newBuilder().setName("add")).addArguments( + Expression.newBuilder().setFunction(Function.newBuilder().setName(FunctionName.newBuilder().setName("manhattanw") ).addArguments( - Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(FEATURE_COL)) + Expression.newBuilder().setColumn(ColumnName.newBuilder().setName(FEATURE_COL)) ).addArguments( - Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) + Expression.newBuilder().setLiteral(Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) ))) ).addArguments( vectorDifference @@ -313,7 +313,7 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i ).addArguments( correctionTerm ) - ).build(); + ).setAlias(ColumnName.newBuilder().setName("distance").build()).build(); return QueryMessage.newBuilder().setQuery( @@ -324,9 +324,7 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i Projection.newBuilder() .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(PERSON_ID_COL))) - .addElements( - distanceFunction - ) + .addElements(distanceFunction) ).setOrder( Order.newBuilder() .addComponents(Component.newBuilder().setColumn(ColumnName.newBuilder().setName(DB_DISTANCE_VALUE_QUALIFIER)) @@ -344,7 +342,7 @@ private static float min(float f, float g, float h) { public static void main(String[] args) throws IOException { - File baseDir = new File("../../Downloads/VBS2022/VBS2022"); + File baseDir = new File("/Users/gassra02/Downloads/VBS2022/"); File[] folders = baseDir.listFiles(File::isDirectory); ObjectMapper mapper = new ObjectMapper(); From 814b088f341ee5708371741cf91d0fca70872f89 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 3 Feb 2022 08:06:01 +0100 Subject: [PATCH 23/72] Optimised INSERT in order to speed them up a little. --- .../cineast/core/features/SkeletonPose.java | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 1061baac4..871dcecf5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -86,21 +86,16 @@ public void initalizePersistentLayer(Supplier supply) { ); } - protected void persist(String shotId, Collection skeletons) { + protected void persist(Collection> skeletons) { if (skeletons == null || skeletons.isEmpty()) { return; } - - List tuples = new ArrayList<>(skeletons.size()); - + final List tuples = new ArrayList<>(skeletons.size()); int i = 0; - for (Skeleton skeleton : skeletons) { - Pair pair = getAnglesandWeights(skeleton); - tuples.add(this.phandler.generateTuple( - shotId, i++, pair.first, pair.second - )); + for (Pair skeleton : skeletons) { + final Pair pair = getAnglesandWeights(skeleton.second); + tuples.add(this.phandler.generateTuple(skeleton.first, i++, pair.first, pair.second)); } - this.phandler.persist(tuples); } @@ -117,8 +112,7 @@ public void processSegment(SegmentContainer segmentContainer) { return; } - persist(segmentContainer.getId(), detectSkeletons(representativeFrame.getImage())); - + this.persist(detectSkeletons(representativeFrame.getImage()).stream().map(it -> new Pair<>(segmentContainer.getId(), it)).collect(Collectors.toList())); } @Override @@ -358,26 +352,30 @@ public static void main(String[] args) throws IOException { SkeletonPose sp = new SkeletonPose(); - + int batchSize = 10000; boolean insert = false; if (insert) { sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); sp.init(() -> new CottontailWriter(ctWrapper), 100); - - + final List> skeletons = new LinkedList<>(); for (File folder : folders) { for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { - - String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); - - List derialized = mapper.readValue(file, typeRef); - - List skeletons = derialized.stream().map(SkeletonEntry::toSkeleton).collect(Collectors.toList()); - - sp.persist(segmentId, skeletons); - + final String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); + for (SkeletonEntry e : mapper.readValue(file, typeRef)) { + skeletons.add(new Pair<>(segmentId, e.toSkeleton())); + } + } + if (skeletons.size() >= batchSize) { + sp.persist(skeletons); + System.out.println("Persisted " + skeletons.size() + " entries..."); + skeletons.clear(); } - System.out.println("done with " + folder.getName()); + } + + /* Final persist. */ + if (skeletons.size() > 0) { + sp.persist(skeletons); + System.out.println("Persisted " + skeletons.size() + " entries..."); } } From f929ebda2fb26e4be7595e8ea9367ab481a62b25 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Fri, 4 Feb 2022 09:17:34 +0100 Subject: [PATCH 24/72] Added todos and fixmes :-) --- .../org/vitrivr/cineast/core/features/SkeletonPose.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 871dcecf5..88d3be03e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -262,7 +262,8 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i for (float w : weights) { queryWeightSum += w; } - + // FIXME: Make it work + // TODO: Cleanup and document Expression vectorDifference = Expression.newBuilder().setFunction(/* Nested, min() function */ CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) @@ -334,9 +335,10 @@ private static float min(float f, float g, float h) { return Math.min(f, Math.min(g, h)); } + // TODO or FIXME: Remove public static void main(String[] args) throws IOException { - File baseDir = new File("/Users/gassra02/Downloads/VBS2022/"); + File baseDir = new File("/Users/rgasser/Downloads/VBS2022/"); File[] folders = baseDir.listFiles(File::isDirectory); ObjectMapper mapper = new ObjectMapper(); From 14cf6273173bd0e704df398bb6221b859eaa1c42 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 8 Feb 2022 11:18:41 +0100 Subject: [PATCH 25/72] Fixed post-merge issues --- .../db/cottontaildb/CottontailSelector.java | 67 +++++++++++++------ .../cineast/core/features/SkeletonPose.java | 38 +++-------- 2 files changed, 56 insertions(+), 49 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 2a30b2775..cf57f34e0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -1,23 +1,6 @@ package org.vitrivr.cineast.core.db.cottontaildb; -import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; -import static org.vitrivr.cineast.core.util.CineastConstants.DOMAIN_COL_NAME; -import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import static org.vitrivr.cineast.core.util.CineastConstants.KEY_COL_NAME; - import io.grpc.StatusRuntimeException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.tuple.Triple; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -38,6 +21,13 @@ import org.vitrivr.cottontail.client.language.extensions.Or; import org.vitrivr.cottontail.client.language.extensions.Predicate; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.vitrivr.cineast.core.util.CineastConstants.*; + public final class CottontailSelector implements DBSelector { /** @@ -60,6 +50,10 @@ public boolean open(String name) { return true; } + public CottontailWrapper getWrapper() { + return this.cottontail; + } + @Override public void close() { /* No op. */ } @@ -258,13 +252,13 @@ public Optional generateQueryFromMetadataSpec(List> reduce = atomics.stream().reduce((res, el) -> { - if (!res.isPresent() && !el.isPresent()) { + if (res.isEmpty() && el.isEmpty()) { return Optional.empty(); } - if (!res.isPresent()) { + if (res.isEmpty()) { return el; } - if (!el.isPresent()) { + if (el.isEmpty()) { return res; } return Optional.of(new Or(res.get(), el.get())); @@ -440,7 +434,38 @@ private static List handleNearestNeighbourRespons try { final Tuple t = response.next(); final String id = t.asString(GENERIC_ID_COLUMN_QUALIFIER); - double distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); /* This should be fine. */ + + double distance = Double.POSITIVE_INFINITY; + + switch (t.type(DB_DISTANCE_VALUE_QUALIFIER)) { + + case BOOLEAN: { + distance = Boolean.TRUE.equals(t.asBoolean(DB_DISTANCE_VALUE_QUALIFIER)) ? 1d : 0d; + break; + } + case BYTE: { + distance = t.asByte(DB_DISTANCE_VALUE_QUALIFIER); + break; + } + case SHORT: + distance = t.asShort(DB_DISTANCE_VALUE_QUALIFIER); + break; + case INTEGER: + distance = t.asInt(DB_DISTANCE_VALUE_QUALIFIER); + break; + case LONG: + distance = t.asLong(DB_DISTANCE_VALUE_QUALIFIER); + break; + case FLOAT: + distance = t.asFloat(DB_DISTANCE_VALUE_QUALIFIER); + break; + case DOUBLE: + distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); + break; + + } + + T e = DistanceElement.create(distanceElementClass, id, distance); result.add(e); } catch (NullPointerException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 88d3be03e..84a242bc0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -31,8 +31,9 @@ import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.grpc.CottontailGrpc; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Expression; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Function; +import org.vitrivr.cottontail.grpc.CottontailGrpc.*; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Component; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Direction; import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; import java.io.File; @@ -40,23 +41,6 @@ import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; -import org.vitrivr.cottontail.grpc.CottontailGrpc.EntityName; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Expression; -import org.vitrivr.cottontail.grpc.CottontailGrpc.FloatVector; -import org.vitrivr.cottontail.grpc.CottontailGrpc.From; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Function; -import org.vitrivr.cottontail.grpc.CottontailGrpc.FunctionName; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Order; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Component; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Direction; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Query; -import org.vitrivr.cottontail.grpc.CottontailGrpc.QueryMessage; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Scan; -import org.vitrivr.cottontail.grpc.CottontailGrpc.SchemaName; import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; @@ -68,12 +52,12 @@ public class SkeletonPose extends AbstractFeatureModule { private static final String WEIGHT_COL = "weights"; public SkeletonPose() { - super("feature_skeletonpose", 1, 8); + super("feature_skeletonpose", (float) (16 * Math.PI), 8); } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - super.init(phandlerSupply, batchSize); + public void init(PersistencyWriterSupplier phandlerSupply) { + super.init(phandlerSupply); this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, PERSON_ID_COL, FEATURE_COL, WEIGHT_COL); } @@ -128,7 +112,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc return Collections.emptyList(); } - SimpleClient client = ((CottontailSelector) this.selector).cottontail.client; + SimpleClient client = ((CottontailSelector) this.selector).getWrapper().client; HashMap>> segmentDistancesMap = new HashMap<>(qc.getRawResultsPerModule() * skeletons.size()); @@ -147,9 +131,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); - int i = 0; while (tuples.hasNext()) { - i++; Tuple tuple = tuples.next(); String segment = tuple.asString(GENERIC_ID_COLUMN_QUALIFIER); @@ -338,7 +320,7 @@ private static float min(float f, float g, float h) { // TODO or FIXME: Remove public static void main(String[] args) throws IOException { - File baseDir = new File("/Users/rgasser/Downloads/VBS2022/"); + File baseDir = new File("../../Downloads/VBS2022/VBS2022"); File[] folders = baseDir.listFiles(File::isDirectory); ObjectMapper mapper = new ObjectMapper(); @@ -349,7 +331,7 @@ public static void main(String[] args) throws IOException { config.setHost("localhost"); config.setPort(1865); - CottontailWrapper ctWrapper = new CottontailWrapper(config, true); + CottontailWrapper ctWrapper = new CottontailWrapper("localhost", 1865); SkeletonPose sp = new SkeletonPose(); @@ -358,7 +340,7 @@ public static void main(String[] args) throws IOException { boolean insert = false; if (insert) { sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); - sp.init(() -> new CottontailWriter(ctWrapper), 100); + sp.init(() -> new CottontailWriter(ctWrapper, 100)); final List> skeletons = new LinkedList<>(); for (File folder : folders) { for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { From 796713906eb317c03a810e725110348947f3ed31 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 8 Feb 2022 12:10:39 +0100 Subject: [PATCH 26/72] Added quantization to weights and fixed sort order in SkeletonPose --- .../org/vitrivr/cineast/core/features/SkeletonPose.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 84a242bc0..c020156cd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -157,7 +157,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist))); } - results.sort(SegmentScoreElement.SCORE_COMPARATOR); + results.sort(SegmentScoreElement.SCORE_COMPARATOR.reversed()); return results.subList(0, Math.min(results.size(), qc.getRawResultsPerModule()) - 1); } @@ -233,6 +233,10 @@ private Pair getAnglesandWeights(Skeleton skeleton) { min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_WRIST)), }; + for (int i = 0; i < weights.length; ++i) { + weights[i] = weights[i] >= 0.5f ? 1f : 0f; + } + return new Pair<>(angles, weights); } @@ -366,7 +370,7 @@ public static void main(String[] args) throws IOException { sp.init(() -> new CottontailSelector(ctWrapper)); - Skeleton skeleton = mapper.readValue(new File(baseDir, "00006/shot00006_22_RKF.json"), typeRef).get(0).toSkeleton(); + Skeleton skeleton = mapper.readValue(new File(baseDir, "00001/shot00001_10_RKF.json"), typeRef).get(0).toSkeleton(); SegmentContainer container = new SegmentContainer() { @Override From 84888c5f2f0daae68db87421e8a262545b460c52 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Wed, 9 Feb 2022 17:53:16 +0100 Subject: [PATCH 27/72] moving wrapper-creation to trace --- .../vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 93f7ed01e..d1283b8f1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -92,7 +92,7 @@ public CottontailWrapper(String host, int port) { boolean pingSuccessful = this.client.ping(); watch.stop(); if (pingSuccessful) { - LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); + LOGGER.trace("Created wrapper in {} ms to {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); } else { LOGGER.warn("Could not connect to Cottontail at {}:{}", host, port); } From 4e6740ecb72ef7d7eff4f20f9bb071b3d4be8b91 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Wed, 9 Feb 2022 18:23:13 +0100 Subject: [PATCH 28/72] fixing npe if maxLength is not specified --- .../org/vitrivr/cineast/core/temporal/TemporalScoring.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/temporal/TemporalScoring.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/temporal/TemporalScoring.java index 542f48602..40b054fe1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/temporal/TemporalScoring.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/temporal/TemporalScoring.java @@ -42,9 +42,9 @@ public TemporalScoring(Map segmentMap, List score() { AbstractTemporalScoringAlgorithm temporalScoringAlgorithm; - float maxLength = this.maxLength; - if (maxLength < 0) { - maxLength = Float.MAX_VALUE; + float maxLength = Float.MAX_VALUE; + if (this.maxLength != null && this.maxLength > 0) { + maxLength = this.maxLength; } if (this.timeDistances.size() > 0) { temporalScoringAlgorithm = new TimeDistanceTemporalScoringAlgorithm(this.segmentMap, this.containerResults, maxLength, this.timeDistances); From 94dce826fece78418361d5b3dc542f0264798a60 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Thu, 10 Feb 2022 22:46:47 +0100 Subject: [PATCH 29/72] reasonable defaults for query parameters --- .../vitrivr/cineast/api/messages/query/TemporalQuery.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java index f3aabd0c2..53f81ebf5 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; import java.util.List; +import kotlin.collections.ArrayDeque; import org.vitrivr.cineast.api.messages.interfaces.MessageType; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; @@ -39,8 +41,8 @@ public TemporalQuery( ) { super(config); this.queries = queries; - this.timeDistances = timeDistances; - this.maxLength = maxLength; + this.timeDistances = timeDistances == null ? new ArrayList<>() : timeDistances; + this.maxLength = maxLength == null ? Float.MAX_VALUE : maxLength; this.metadataAccessSpec = metadataAccessSpec; } From 178a16a04f9a87f9d5db09ae4560221e85f61078 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Fri, 4 Mar 2022 10:20:29 +0100 Subject: [PATCH 30/72] Fixed JSON deserialization of Skeleton. --- .../main/java/org/vitrivr/cineast/core/data/Skeleton.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java index a0af43ca8..e0c226774 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -1,5 +1,7 @@ package org.vitrivr.cineast.core.data; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import georegression.struct.point.Point2D_F32; import java.util.ArrayList; @@ -40,7 +42,11 @@ public enum SkeletonPointName { private final Point2D_F32[] points = new Point2D_F32[POINT_COUNT]; private final float[] weights = new float[POINT_COUNT]; - public Skeleton(float[] coordinates, float[] weights) { + @JsonCreator + public Skeleton( + @JsonProperty("coordinates") float[] coordinates, + @JsonProperty("weights") float[] weights + ) { if (coordinates == null || coordinates.length < 2 * POINT_COUNT || weights == null || weights.length < POINT_COUNT) { throw new IllegalArgumentException(); From f872bf467426aedbd69f5a35aa7ac285642d1e24 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Mon, 7 Mar 2022 20:48:28 +0100 Subject: [PATCH 31/72] adding providedOCR feature --- .../core/features/ProvidedOcrSearch.java | 18 ++++++++++++++++++ .../standalone/cli/TextRetrievalCommand.java | 1 - 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/ProvidedOcrSearch.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ProvidedOcrSearch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ProvidedOcrSearch.java new file mode 100644 index 000000000..c54159b62 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ProvidedOcrSearch.java @@ -0,0 +1,18 @@ +package org.vitrivr.cineast.core.features; + +import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; + +/** + * Used when OCR is provided by an external API, e.g. Google Vision + */ +public class ProvidedOcrSearch extends AbstractTextRetriever { + + public static final String PROVIDED_OCR_SEARCH_TABLE_NAME = "features_providedOcr"; + + /** + * Default constructor for {@link ProvidedOcrSearch}. + */ + public ProvidedOcrSearch() { + super(ProvidedOcrSearch.PROVIDED_OCR_SEARCH_TABLE_NAME); + } +} \ No newline at end of file diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java index 6eeb3a2cf..83354309f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java @@ -35,7 +35,6 @@ public void run() { retrievers.add(new OCRSearch()); retrievers.add(new AudioTranscriptionSearch()); retrievers.add(new DescriptionTextSearch()); - retrievers.add(new TagsFtSearch()); CliUtils.retrieveAndLog(retrievers, retrieval, limit, printDetail, qc); System.out.println("Done"); } From 341ac14558323e8fc5eb56250c773c79a1b829bd Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Tue, 8 Mar 2022 16:54:18 +0100 Subject: [PATCH 32/72] disabling transactions for cottontail import, logging improvements --- .../cineast/core/db/cottontaildb/CottontailWriter.java | 8 ++++---- .../org/vitrivr/cineast/standalone/cli/ImportCommand.java | 2 +- .../importer/handlers/JsonDataImportHandler.java | 2 +- .../cineast/standalone/monitoring/ImportTaskMonitor.java | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index 3f7c8eed3..6eaafa3c2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -57,9 +57,9 @@ public boolean exists(String key, String value) { public boolean persist(List tuples) { long start = System.currentTimeMillis(); int size = tuples.size(); - final long txId = this.cottontail.client.begin(); + //final long txId = this.cottontail.client.begin(); try { - BatchInsert insert = new BatchInsert().into(this.fqn).columns(this.names).txId(txId); + BatchInsert insert = new BatchInsert().into(this.fqn).columns(this.names);//.txId(txId); while (!tuples.isEmpty()) { final PersistentTuple tuple = tuples.remove(0); final Object[] values = tuple.getElements().stream().map(o -> { @@ -80,12 +80,12 @@ public boolean persist(List tuples) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); this.cottontail.client.insert(insert); } - this.cottontail.client.commit(txId); + //this.cottontail.client.commit(txId); long stop = System.currentTimeMillis(); LOGGER.trace("Completed insert of {} elements in {} ms", size, stop - start); return true; } catch (StatusRuntimeException e) { - this.cottontail.client.rollback(txId); + //this.cottontail.client.rollback(txId); return false; } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java index 31276ed2f..7513a5051 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java @@ -60,7 +60,7 @@ public class ImportCommand implements Runnable { @Override public void run() { - System.out.printf("Starting import of type %s for '%s'.%n", this.type, this.input); + System.out.printf("Starting import of type %s for '%s'. Batchsize %d, %d threads. Clean %b, no-finalize %b .%n", this.type, this.input, this.batchsize, this.threads, this.clean, this.doNotFinalize); final Path path = Paths.get(this.input); final ImportType type = ImportType.valueOf(this.type.toUpperCase()); DataImportHandler handler = null; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java index db58a7ea2..857134383 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/JsonDataImportHandler.java @@ -28,7 +28,7 @@ public JsonDataImportHandler(int threads, int batchsize) { public void doImport(Path path) { try { LOGGER.info("Starting data import with JSON files in: {}", path.toString()); - Files.walk(path, 2).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> { + Files.walk(path, 3).filter(p -> p.toString().toLowerCase().endsWith(".json")).forEach(p -> { final String filename = p.getFileName().toString(); final String suffix = filename.substring(filename.lastIndexOf(".")); try { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java index 9391c9468..3f0bd3b27 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/ImportTaskMonitor.java @@ -49,7 +49,7 @@ public static void reportExecutionTime(String name, long milliseconds) { * You can call this method without worrying if prometheus support is enabled */ public static void reportImportProgress(int insertedTuples, String entityName, long executiontime) { - LOGGER.trace("Completed import of {} tuples on {} in {} ms", insertedTuples, entityName, executiontime); + LOGGER.trace("import progress: {} tuples on {} in {} ms", insertedTuples, entityName, executiontime); if (importProgress != null && batchImportTime != null) { importProgress.labels(entityName).inc(insertedTuples); batchImportTime.labels(entityName, String.valueOf(insertedTuples)).observe(executiontime); From 899dd45c1d61cd336e8fdabc65cbff049e12af8b Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Fri, 11 Mar 2022 16:05:26 +0100 Subject: [PATCH 33/72] Some hotfixes to get things working again --- .../vitrivr/cineast/api/messages/query/TemporalQuery.java | 7 ++++++- .../api/rest/resolvers/FileSystemThumbnailResolver.java | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java index 19b188433..9fab868ab 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java @@ -2,10 +2,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; import org.vitrivr.cineast.api.messages.interfaces.MessageType; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; +import java.util.List; + /** * This object represents a temporal-query message of temporal query version 2, i.e. a request for a temporally staged similarity-search. */ @@ -76,4 +77,8 @@ public List getMetadataAccessSpec() { public MessageType getMessageType() { return MessageType.Q_TEMPORAL; } + + public void setTimeDistances(List distances) { + //FIXME there is a timeDistances field in the request that should presumably not be there. This is just a hotfix to get the message to serialize again. + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java index 43236dc9a..ed1043429 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java @@ -24,9 +24,9 @@ public ResolutionResult resolve(String segmentId) { return null; } - String fileName = segmentId.substring(0, segmentId.lastIndexOf("_")); + //String fileName = segmentId.substring(0, segmentId.lastIndexOf("_")); - File dir = new File(this.baseFolder, fileName); + File dir = new File(this.baseFolder, split[1]); if (!dir.exists() || !dir.isDirectory()) { return null; @@ -35,7 +35,7 @@ public ResolutionResult resolve(String segmentId) { File[] candidates = dir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return name.startsWith(segmentId); + return name.startsWith("shot" + split[1] + "_" + split[2]); } }); From a2a1f672854516495eaa79c756939bff90fbf104 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 14 Mar 2022 14:27:11 +0100 Subject: [PATCH 34/72] Added more angles to SkeletonPose feature and cleaned up a bit --- .../cineast/core/features/SkeletonPose.java | 143 +++++++++++------- 1 file changed, 89 insertions(+), 54 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index c020156cd..8c0b21401 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -52,7 +52,7 @@ public class SkeletonPose extends AbstractFeatureModule { private static final String WEIGHT_COL = "weights"; public SkeletonPose() { - super("feature_skeletonpose", (float) (16 * Math.PI), 8); + super("feature_skeletonpose", (float) (16 * Math.PI), 12); } @Override @@ -122,15 +122,8 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc for (Skeleton skeleton : skeletons) { Pair pair = getAnglesandWeights(skeleton); - -// List> rows = this.selector.getNearestNeighbourRows(qc.getRawResultsPerModule(), pair.first, FEATURE_COL, QueryConfig.clone(qc) -// .setDistanceWeights(pair.second) -// .setDistance(ReadableQueryConfig.Distance.manhattan)); - - TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); - while (tuples.hasNext()) { Tuple tuple = tuples.next(); @@ -210,27 +203,39 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc private Pair getAnglesandWeights(Skeleton skeleton) { float[] angles = new float[]{ - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ANKLE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ANKLE)), - - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_WRIST), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW)), - angle(skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getPoint(Skeleton.SkeletonPointName.RIGHT_WRIST)) + angle(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), + + angle(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), + + angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + + angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) }; float[] weights = new float[]{ - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ANKLE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_HIP), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_KNEE), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ANKLE)), - - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_WRIST), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.LEFT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW)), - min(skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_SHOULDER), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_ELBOW), skeleton.getWeight(Skeleton.SkeletonPointName.RIGHT_WRIST)), + min(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), + min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), + min(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), + + min(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), + + min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + + min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) }; for (int i = 0; i < weights.length; ++i) { @@ -241,6 +246,28 @@ private Pair getAnglesandWeights(Skeleton skeleton) { } + private static Expression expression(float f) { + return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData(f)).build(); + } + + private static Expression expression(float[] f) { + return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(f)) + ))).build(); + } + + private static Function.Builder functionBuilder(String name) { + return CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName(name)); + } + + private static ColumnName columnName(String name) { + return ColumnName.newBuilder().setName(name).build(); + } + + private static ProjectionElement projectionElement(String name) { + return ProjectionElement.newBuilder().setColumn(columnName(name)).build(); + } + private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { float queryWeightSum = 0f; @@ -248,53 +275,53 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i for (float w : weights) { queryWeightSum += w; } - // FIXME: Make it work - // TODO: Cleanup and document + + // element-wise difference between stored and provided weights + // counts how many elements are set in the query but are not set in the feature Expression vectorDifference = Expression.newBuilder().setFunction(/* Nested, min() function */ - CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vmin")).addArguments( - Expression.newBuilder().setColumn(CottontailGrpc.ColumnName.newBuilder().setName(WEIGHT_COL)) + functionBuilder("vmin").addArguments( + Expression.newBuilder().setColumn(columnName(WEIGHT_COL)) ).addArguments( - Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(weights)) - ))) + expression(weights) ) ).build(); + //assigns maximum distance for each element specified in the query but not present in the feature Expression correctionTerm = Expression.newBuilder().setFunction( - Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("mul") - ).addArguments( //constant - Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData((float) Math.PI)) + functionBuilder("mul") + .addArguments( //constant + expression((float) Math.PI) ).addArguments( //sub-expression Expression.newBuilder().setFunction( Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") ).addArguments( - Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData(queryWeightSum)) + expression(queryWeightSum) ).addArguments( - Expression.newBuilder().setFunction(CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("vsum")).addArguments(vectorDifference)) + Expression.newBuilder().setFunction( + functionBuilder("vsum") + .addArguments(vectorDifference) + ) ) ) ) ).build(); - + //weighted Manhattan-distance plus correction term for missing elements ProjectionElement distanceFunction = ProjectionElement.newBuilder().setFunction(/* Distance function */ - - Function.newBuilder().setName(FunctionName.newBuilder().setName("add")).addArguments( - Expression.newBuilder().setFunction(Function.newBuilder().setName(FunctionName.newBuilder().setName("manhattanw") + functionBuilder("add").addArguments( + Expression.newBuilder().setFunction(functionBuilder("manhattanw") + .addArguments( + Expression.newBuilder().setColumn(columnName(FEATURE_COL)) ).addArguments( - Expression.newBuilder().setColumn(ColumnName.newBuilder().setName(FEATURE_COL)) - ).addArguments( - Expression.newBuilder().setLiteral(Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - FloatVector.newBuilder().addAllVector(new FloatArrayIterable(query)) - ))) + expression(query) ).addArguments( vectorDifference )) ).addArguments( correctionTerm ) - ).setAlias(ColumnName.newBuilder().setName("distance").build()).build(); + ).setAlias(columnName("distance")).build(); return QueryMessage.newBuilder().setQuery( @@ -303,24 +330,32 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i .setName(this.tableName).setSchema(SchemaName.newBuilder().setName("cineast")))) ).setProjection( Projection.newBuilder() - .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(GENERIC_ID_COLUMN_QUALIFIER))) - .addElements(ProjectionElement.newBuilder().setColumn(ColumnName.newBuilder().setName(PERSON_ID_COL))) + .addElements(projectionElement(GENERIC_ID_COLUMN_QUALIFIER)) + .addElements(projectionElement(PERSON_ID_COL)) .addElements(distanceFunction) ).setOrder( Order.newBuilder() - .addComponents(Component.newBuilder().setColumn(ColumnName.newBuilder().setName(DB_DISTANCE_VALUE_QUALIFIER)) - .setDirection(Direction.ASCENDING).build()).build() - ).setLimit(limit).build()).build(); + .addComponents(Component.newBuilder().setColumn(columnName(DB_DISTANCE_VALUE_QUALIFIER)) + .setDirection(Direction.ASCENDING)).build() + ).setLimit(limit)).build(); } private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { return (float) (Math.atan2(p2.y - c.y, p2.x - c.x) - Math.atan2(p1.y - c.y, p1.x - c.x)); } + private static float angle(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { + return angle(skeleton.getPoint(p1), skeleton.getPoint(p2), skeleton.getPoint(p3)); + } + private static float min(float f, float g, float h) { return Math.min(f, Math.min(g, h)); } + private static float min(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { + return min(skeleton.getWeight(p1), skeleton.getWeight(p2), skeleton.getWeight(p3)); + } + // TODO or FIXME: Remove public static void main(String[] args) throws IOException { @@ -341,7 +376,7 @@ public static void main(String[] args) throws IOException { SkeletonPose sp = new SkeletonPose(); int batchSize = 10000; - boolean insert = false; + boolean insert = true; if (insert) { sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); sp.init(() -> new CottontailWriter(ctWrapper, 100)); From a102100a5003b7ef35b3b2e8883900c06b879c55 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Mon, 14 Mar 2022 20:59:19 +0100 Subject: [PATCH 35/72] reformatting imports --- .../vitrivr/cineast/api/messages/query/QueryTermType.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java index 3b1e8397d..4186a74a7 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTermType.java @@ -1,9 +1,5 @@ package org.vitrivr.cineast.api.messages.query; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.query.containers.*; - import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Optional; @@ -19,6 +15,7 @@ import org.vitrivr.cineast.core.data.query.containers.ModelQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.ParameterisedLocationQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.SemanticMapQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.SkeletonQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.TagQueryTermContainer; import org.vitrivr.cineast.core.data.query.containers.TextQueryTermContainer; From 11e1795a76560f59ff891f54a88146986a756be1 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Mon, 14 Mar 2022 21:11:08 +0100 Subject: [PATCH 36/72] fixing main --- .../java/org/vitrivr/cineast/core/features/SkeletonPose.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 8c0b21401..dcf02a38f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -370,7 +370,7 @@ public static void main(String[] args) throws IOException { config.setHost("localhost"); config.setPort(1865); - CottontailWrapper ctWrapper = new CottontailWrapper("localhost", 1865); + final CottontailWrapper ctWrapper = new CottontailWrapper("localhost", 1865); SkeletonPose sp = new SkeletonPose(); @@ -379,7 +379,7 @@ public static void main(String[] args) throws IOException { boolean insert = true; if (insert) { sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); - sp.init(() -> new CottontailWriter(ctWrapper, 100)); + sp.init(() -> new CottontailWriter(ctWrapper, 100, true)); final List> skeletons = new LinkedList<>(); for (File folder : folders) { for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { From 0cbbc088490f29bb7ba702e037d3641b111c379d Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 16 Mar 2022 10:43:28 +0100 Subject: [PATCH 37/72] Added OpenPose-based pose detector --- .../cineast/core/features/SkeletonPose.java | 7 +- .../core/util/pose/OpenPoseDetector.java | 351 ++++++++++++++++++ 2 files changed, 355 insertions(+), 3 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index dcf02a38f..af7552280 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -5,7 +5,6 @@ import georegression.struct.point.Point2D_F32; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TObjectDoubleHashMap; -import org.apache.commons.lang3.NotImplementedException; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; @@ -27,6 +26,7 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.HungarianAlgorithm; +import org.vitrivr.cineast.core.util.pose.OpenPoseDetector; import org.vitrivr.cottontail.client.SimpleClient; import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; @@ -50,6 +50,7 @@ public class SkeletonPose extends AbstractFeatureModule { private static final String PERSON_ID_COL = "person"; private static final String FEATURE_COL = "skeleton"; private static final String WEIGHT_COL = "weights"; + private OpenPoseDetector detector = new OpenPoseDetector(); public SkeletonPose() { super("feature_skeletonpose", (float) (16 * Math.PI), 12); @@ -83,8 +84,8 @@ protected void persist(Collection> skeletons) { this.phandler.persist(tuples); } - private List detectSkeletons(MultiImage img) { - throw new NotImplementedException("not currently available"); + private synchronized List detectSkeletons(MultiImage img) { + return detector.recognizePoses(img.getBufferedImage()); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java new file mode 100644 index 000000000..596ce7201 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java @@ -0,0 +1,351 @@ +package org.vitrivr.cineast.core.util.pose; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.tensorflow.Graph; +import org.tensorflow.Session; +import org.tensorflow.ndarray.Shape; +import org.tensorflow.ndarray.buffer.DataBuffers; +import org.tensorflow.ndarray.buffer.FloatDataBuffer; +import org.tensorflow.proto.framework.GraphDef; +import org.tensorflow.types.TFloat32; +import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.util.LogHelper; +import org.vitrivr.cineast.core.util.images.ImagePreprocessingHelper; + +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +/** + * Based on https://github.com/liljom/openpose-tf-mobilenet-java + */ +public class OpenPoseDetector implements AutoCloseable +{ + private static final String MODEL_FILE = "resources/thin.pb"; + private static final String INPUT_NAME = "image"; + private static final String OUTPUT_NAME = "Openpose/concat_stage7"; + private static final int[][] COCO_PAIRS = {{1, 2}, {1, 5}, {2, 3}, {3, 4}, {5, 6}, {6, 7}, {1, 8}, {8, 9}, {9, 10}, {1, 11}, + {11, 12}, {12, 13}, {1, 0}, {0, 14}, {14, 16}, {0, 15}, {15, 17}}; + private final int[][] COCO_PAIR_CONNECTIONS = {{12, 13}, {20, 21}, {14, 15}, {16, 17}, {22, 23}, {24, 25}, {0, 1}, {2, 3}, + {4, 5}, {6, 7}, {8, 9}, {10, 11}, {28, 29}, {30, 31}, {34, 35}, {32, 33}, {36, 37}, {18, 19}, {26, 27}}; + private final float[] STD = new float[]{1f/255f, 1f/255f, 1f/255f}; + + + private final Graph graph; + private final Session session; + + protected static byte[] load(String path) { + try { + return Files.readAllBytes((Paths.get(path))); + } catch (IOException e) { + throw new RuntimeException( + "could not load graph for DeepLab: " + LogHelper.getStackTrace(e)); + } + } + + public OpenPoseDetector() + { + this.graph = new Graph(); + try { + this.graph.importGraphDef(GraphDef.parseFrom(load(MODEL_FILE))); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + this.session = new Session(this.graph); + + } + + public void close() { + this.session.close(); + this.graph.close(); + } + + public List recognizePoses(BufferedImage img) + { + + final int imageSize = 512; + float scaling = ((float) imageSize) / Math.max(img.getWidth(), img.getHeight()); + int xOffset = (int) ((imageSize - (img.getWidth() * scaling)) / 2f); + int yOffset = (int) ((imageSize - (img.getHeight() * scaling)) / 2f); + + BufferedImage resizedImg; + if (img.getWidth() == imageSize && img.getHeight() == imageSize) { + resizedImg = img; + } else { + resizedImg = new BufferedImage(imageSize, imageSize, BufferedImage.TYPE_INT_RGB); + Graphics2D g2 = resizedImg.createGraphics(); + g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g2.setColor(Color.white); + g2.fillRect(0, 0, imageSize, imageSize); + g2.drawImage(img, xOffset, yOffset, (int)(img.getWidth() * scaling), (int)(img.getHeight() * scaling), null); + g2.dispose(); + } + + float[] fimg = ImagePreprocessingHelper.imageToHWCArray(resizedImg, null, STD); + + TFloat32 imageTensor = TFloat32.tensorOf(Shape.of(1, imageSize, imageSize, 3), DataBuffers.of(fimg)); + + TFloat32 out = (TFloat32) session.runner().feed(INPUT_NAME, imageTensor).fetch(OUTPUT_NAME).run().get(0); + float[] outputTensor = new float[(int) out.size()]; + + FloatDataBuffer floatBuffer = DataBuffers.of(outputTensor); + out.read(floatBuffer); + + int heatMapCount = 19; + List> coordinates = new ArrayList<>(heatMapCount - 1); + + int mapHeight = imageSize / 8; + int mapWidth = imageSize / 8; + + // eliminate duplicate part recognitions + final int pafMapCount = 38; + for (int i = 0; i < (heatMapCount - 1); i++) { + coordinates.add(new Vector<>()); + for (int j = 0; j < mapHeight; j++) { + for (int k = 0; k < mapWidth; k++) { + float[] coordinate = {j, k}; + float max_value = 0; + int maximumFilterSize = 5; + for (int dj = -(maximumFilterSize - 1) / 2; dj < (maximumFilterSize + 1) / 2; dj++) { + if ((j + dj) >= mapHeight || (j + dj) < 0) { + break; + } + for (int dk = -(maximumFilterSize - 1) / 2; dk < (maximumFilterSize + 1) / 2; dk++) { + if ((k + dk) >= mapWidth || (k + dk) < 0) { + break; + } + float value = outputTensor[(heatMapCount + pafMapCount) * mapWidth * (j + dj) + (heatMapCount + pafMapCount) * (k + dk) + i]; + if (value > max_value) { + max_value = value; + } + } + } + final float NMS_Threshold = 0.15f; + if (max_value > NMS_Threshold) { + if (max_value == outputTensor[(heatMapCount + pafMapCount) * mapWidth * j + (heatMapCount + pafMapCount) * k + i]) { + coordinates.get(i).addElement(coordinate); + } + } + } + } + } + + // eliminate duplicate connections + final int maxPairCount = 17; + List> pairs = new ArrayList<>(maxPairCount); + List> pairs_final = new ArrayList<>(maxPairCount); + List> pairs_scores = new ArrayList<>(maxPairCount); + List> pairs_scores_final = new ArrayList<>(maxPairCount); + for (int i = 0; i < maxPairCount; i++) { + pairs.add(new Vector<>()); + pairs_scores.add(new Vector<>()); + pairs_final.add(new Vector<>()); + pairs_scores_final.add(new Vector<>()); + Vector part_set = new Vector<>(); + for (int p1 = 0; p1 < coordinates.get(COCO_PAIRS[i][0]).size(); p1++) { + for (int p2 = 0; p2 < coordinates.get(COCO_PAIRS[i][1]).size(); p2++) { + int count = 0; + float score = 0.0f; + float[] scores = new float[10]; + float p1x = coordinates.get(COCO_PAIRS[i][0]).get(p1)[0]; + float p1y = coordinates.get(COCO_PAIRS[i][0]).get(p1)[1]; + float p2x = coordinates.get(COCO_PAIRS[i][1]).get(p2)[0]; + float p2y = coordinates.get(COCO_PAIRS[i][1]).get(p2)[1]; + float dx = p2x - p1x; + float dy = p2y - p1y; + float normVec = (float) Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2)); + + if (normVec < 0.0001f) { + break; + } + float vx = dx / normVec; + float vy = dy / normVec; + for (int t = 0; t < 10; t++) { + int tx = (int) (p1x + (t * dx / 9) + 0.5); + int ty = (int) (p1y + (t * dy / 9) + 0.5); + int location = tx * (heatMapCount + pafMapCount) * mapWidth + ty * (heatMapCount + pafMapCount) + heatMapCount; + scores[t] = vy * outputTensor[location + COCO_PAIR_CONNECTIONS[i][0]]; + scores[t] += vx * outputTensor[location + COCO_PAIR_CONNECTIONS[i][1]]; + } + for (int h = 0; h < 10; h++) { + float local_PAF_Threshold = 0.2f; + if (scores[h] > local_PAF_Threshold) { + count += 1; + score += scores[h]; + } + } + float part_Score_Threshold = 0.2f; + int PAF_Count_Threshold = 5; + if (score > part_Score_Threshold && count >= PAF_Count_Threshold) { + boolean inserted = false; + int[] pair = {p1, p2}; + for (int l = 0; l < pairs.get(i).size(); l++) { + if (score > pairs_scores.get(i).get(l)) { + pairs.get(i).insertElementAt(pair, l); + pairs_scores.get(i).insertElementAt(score, l); + inserted = true; + break; + } + } + if (!inserted) { + pairs.get(i).addElement(pair); + pairs_scores.get(i).addElement(score); + } + } + } + } + for (int m = 0; m < pairs.get(i).size(); m++) { + boolean conflict = false; + for (Integer integer : part_set) { + if (pairs.get(i).get(m)[0] == integer || pairs.get(i).get(m)[1] == integer) { + conflict = true; + break; + } + } + if (!conflict) { + pairs_final.get(i).addElement(pairs.get(i).get(m)); + pairs_scores_final.get(i).addElement(pairs_scores.get(i).get(m)); + part_set.addElement(pairs.get(i).get(m)[0]); + part_set.addElement(pairs.get(i).get(m)[1]); + } + } + } + + ArrayList humans = new ArrayList<>(); + ArrayList skeletons = new ArrayList<>(); + for (int i = 0; i < maxPairCount; i++) { + for (int j = 0; j < pairs_final.get(i).size(); j++) { + boolean merged = false; + int p1 = COCO_PAIRS[i][0]; + int p2 = COCO_PAIRS[i][1]; + int ip1 = pairs_final.get(i).get(j)[0]; + int ip2 = pairs_final.get(i).get(j)[1]; + for (Human human : humans) { + if ((ip1 == human.coords_index_set[p1] && human.coords_index_assigned[p1]) || (ip2 == human.coords_index_set[p2] && human.coords_index_assigned[p2])) { + human.parts_coords[p1] = coordinates.get(p1).get(ip1); + human.parts_coords[p2] = coordinates.get(p2).get(ip2); + human.coords_index_set[p1] = ip1; + human.coords_index_set[p2] = ip2; + human.coords_index_assigned[p1] = true; + human.coords_index_assigned[p2] = true; + merged = true; + break; + } + } + if (!merged) { + Human human = new Human(); + human.parts_coords[p1] = coordinates.get(p1).get(ip1); + human.parts_coords[p2] = coordinates.get(p2).get(ip2); + human.coords_index_set[p1] = ip1; + human.coords_index_set[p2] = ip2; + human.coords_index_assigned[p1] = true; + human.coords_index_assigned[p2] = true; + humans.add(human); + } + } + } + + // remove people with too few parts + for (Human human : humans) { + int human_part_count = 0; + for (int j = 0; j < heatMapCount - 1; j++) { + if (human.coords_index_assigned[j]) { + human_part_count += 1; + } + } + int part_Count_Threshold = 4; + if (human_part_count > part_Count_Threshold) { + + Skeleton skeleton = human.toSkeleton(xOffset, yOffset, scaling); + skeletons.add(skeleton); + + } + } + + return skeletons; + } + + +// "nose", //0 +// "neck", //1 +// "rShoulder", //2 +// "rElbow", //3 +// "rWist", //4 +// "lShoulder", //5 +// "lElbow", //6 +// "lWrist", //7 +// "rHip", //8 +// "rKnee", //9 +// "rAnkle", //10 +// "lHip", //11 +// "lKnee", //12 +// "lAnkle", //13 +// "rEye", //14 +// "lEye", //15 +// "rEar", //16 +// "lEar" //17 + public class Human + { + + float[][] parts_coords = new float[18][2]; + int[] coords_index_set = new int[18]; + boolean[] coords_index_assigned = new boolean[18]; + + public Skeleton toSkeleton(float xOffset, float yOffset, float scale) { + + float[] coordinates = { + parts_coords[0][1], parts_coords[0][0], //NOSE + parts_coords[15][1], parts_coords[15][0], //LEFT_EYE + parts_coords[14][1], parts_coords[14][0], //RIGHT_EYE + parts_coords[17][1], parts_coords[17][0], //LEFT_EAR + parts_coords[16][1], parts_coords[16][0], //RIGHT_EAR + parts_coords[5][1], parts_coords[5][0], //LEFT_SHOULDER + parts_coords[2][1], parts_coords[2][0], //RIGHT_SHOULDER + parts_coords[6][1], parts_coords[6][0], //LEFT_ELBOW + parts_coords[3][1], parts_coords[3][0], //RIGHT_ELBOW + parts_coords[7][1], parts_coords[7][0], //LEFT_WRIST + parts_coords[4][1], parts_coords[4][0], //RIGHT_WRIST + parts_coords[11][1], parts_coords[11][0], //LEFT_HIP + parts_coords[8][1], parts_coords[8][0], //RIGHT_HIP + parts_coords[12][1], parts_coords[12][0], //LEFT_KNEE + parts_coords[9][1], parts_coords[9][0], //RIGHT_KNEE + parts_coords[13][1], parts_coords[13][0], //LEFT_ANKLE + parts_coords[10][1], parts_coords[10][0], //RIGHT_ANKLE + }; + + for (int i = 0; i < coordinates.length; i += 2) { + coordinates[i] = (coordinates[i] * 8f - xOffset) / scale; + coordinates[i + 1] = (coordinates[i + 1] * 8f - yOffset) / scale; + } + + float[] weights = { + coords_index_assigned[0] ? 1f : 0f, + coords_index_assigned[15] ? 1f : 0f, + coords_index_assigned[14] ? 1f : 0f, + coords_index_assigned[17] ? 1f : 0f, + coords_index_assigned[16] ? 1f : 0f, + coords_index_assigned[5] ? 1f : 0f, + coords_index_assigned[2] ? 1f : 0f, + coords_index_assigned[6] ? 1f : 0f, + coords_index_assigned[3] ? 1f : 0f, + coords_index_assigned[7] ? 1f : 0f, + coords_index_assigned[4] ? 1f : 0f, + coords_index_assigned[11] ? 1f : 0f, + coords_index_assigned[8] ? 1f : 0f, + coords_index_assigned[12] ? 1f : 0f, + coords_index_assigned[9] ? 1f : 0f, + coords_index_assigned[13] ? 1f : 0f, + coords_index_assigned[10] ? 1f : 0f, + }; + + return new Skeleton(coordinates,weights); + + } + } +} From 391ed57717f9e37f691506ad7a0fb4e5a018b1b3 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 16 Mar 2022 17:44:13 +0100 Subject: [PATCH 38/72] Added pose detector based on Movenet --- .../vitrivr/cineast/core/data/Skeleton.java | 28 +++-- .../util/pose/MovenetMultiposeDetector.java | 110 ++++++++++++++++++ 2 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java index e0c226774..057ee3558 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -29,13 +29,27 @@ public enum SkeletonPointName { RIGHT_ANKLE } - /* - - [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], [5, 11], [6, 12], [5, 6], [5, 7], - [6, 8], [7, 9], [8, 10], [1, 2], [0, 1], [0, 2], [1, 3], [2, 4], - [0, 5], [0, 6] - - */ + public static final SkeletonPointName[][] BONES = { + {SkeletonPointName.LEFT_ANKLE, SkeletonPointName.LEFT_KNEE}, + {SkeletonPointName.LEFT_KNEE, SkeletonPointName.LEFT_HIP}, + {SkeletonPointName.RIGHT_ANKLE, SkeletonPointName.RIGHT_KNEE}, + {SkeletonPointName.RIGHT_KNEE, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_HIP, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_HIP}, + {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.RIGHT_SHOULDER}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_ELBOW}, + {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_ELBOW}, + {SkeletonPointName.LEFT_ELBOW, SkeletonPointName.LEFT_WRIST}, + {SkeletonPointName.RIGHT_ELBOW, SkeletonPointName.RIGHT_WRIST}, + {SkeletonPointName.LEFT_EYE, SkeletonPointName.RIGHT_EYE}, + {SkeletonPointName.NOSE, SkeletonPointName.LEFT_EYE}, + {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_EYE}, + {SkeletonPointName.LEFT_EYE, SkeletonPointName.LEFT_EAR}, + {SkeletonPointName.RIGHT_EYE, SkeletonPointName.RIGHT_EAR}, + {SkeletonPointName.NOSE, SkeletonPointName.LEFT_SHOULDER}, + {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_SHOULDER} + }; private static final int POINT_COUNT = 17; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java new file mode 100644 index 000000000..b2fd21365 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java @@ -0,0 +1,110 @@ +package org.vitrivr.cineast.core.util.pose; + +import org.tensorflow.ConcreteFunction; +import org.tensorflow.SavedModelBundle; +import org.tensorflow.Tensor; +import org.tensorflow.ndarray.Shape; +import org.tensorflow.ndarray.buffer.DataBuffers; +import org.tensorflow.ndarray.buffer.FloatDataBuffer; +import org.tensorflow.types.TFloat32; +import org.tensorflow.types.TInt32; +import org.vitrivr.cineast.core.data.Skeleton; + +import java.awt.*; +import java.awt.image.BufferedImage; +import java.util.*; +import java.util.List; + + +/** + * Detects up to 6 complete poses + * see https://tfhub.dev/google/movenet/multipose/lightning/1 + */ +public class MovenetMultiposeDetector { + + private static final String RESOURCE_PATH = "resources/movenet_multipose_lightning/"; + private static final String FUNCTION = "serving_default"; + + private final SavedModelBundle multiPose; + private final ConcreteFunction function; + + public MovenetMultiposeDetector() { + this.multiPose = SavedModelBundle.load(RESOURCE_PATH); + this.function = this.multiPose.function(FUNCTION); + } + + + public List recognizePoses(BufferedImage img) { + + final int imageSize = 256; + float scaling = ((float) imageSize) / Math.max(img.getWidth(), img.getHeight()); + int xOffset = (int) ((imageSize - (img.getWidth() * scaling)) / 2f); + int yOffset = (int) ((imageSize - (img.getHeight() * scaling)) / 2f); + + BufferedImage resizedImg; + if (img.getWidth() == imageSize && img.getHeight() == imageSize) { + resizedImg = img; + } else { + resizedImg = new BufferedImage(imageSize, imageSize, BufferedImage.TYPE_INT_RGB); + Graphics2D g2 = resizedImg.createGraphics(); + g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g2.setColor(Color.white); + g2.fillRect(0, 0, imageSize, imageSize); + g2.drawImage(img, xOffset, yOffset, (int) (img.getWidth() * scaling), (int) (img.getHeight() * scaling), null); + g2.dispose(); + } + + int[] colors = resizedImg.getRGB(0, 0, imageSize, imageSize, null, 0, imageSize); + int[] rgb = new int[imageSize * imageSize * 3]; + + for (int i = 0; i < colors.length; ++i) { + int j = i * 3; + rgb[j] = (colors[i] >> 16) & 0xFF; // r + rgb[j + 1] = (colors[i] >> 8) & 0xFF; // g + rgb[j + 2] = (colors[i] & 0xFF); //b + } + + float[] points = new float[6 * 56]; + + try (Tensor imageTensor = TInt32.tensorOf(Shape.of(1, imageSize, imageSize, 3), DataBuffers.of(rgb))) { + TFloat32 pointsTensor = (TFloat32) this.function.call(imageTensor); + FloatDataBuffer floatBuffer = DataBuffers.of(points); + pointsTensor.read(floatBuffer); + pointsTensor.close(); + } + + final int resultLength = 56; + + ArrayList skeletons = new ArrayList<>(6); + + for (int pose = 0; pose < 6; ++pose) { + + int offset = pose * resultLength; + + float score = points[offset + 55]; + + if (score < 0.5f) { + continue; + } + + float[] coords = new float[17 * 2]; + float[] weights = new float[17]; + + for (int i = 0; i < 17; ++i) { + coords[2 * i + 1] = ((points[offset + 3 * i] * imageSize) - yOffset) / scaling; + coords[2 * i] = ((points[offset + 3 * i + 1] * imageSize) - xOffset) / scaling; + weights[i] = points[offset + 3 * i + 1]; + } + + skeletons.add( + new Skeleton(coords, weights) + ); + + } + + return skeletons; + + } + +} From df5d9f064ed4b1c3b508542ed2bae22f4af788b4 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Fri, 18 Mar 2022 07:49:35 +0100 Subject: [PATCH 39/72] Added caching to FileSystemObjectResolver --- .../resolvers/FileSystemObjectResolver.java | 93 ++++++++++++------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java index 426280f76..f3e6ced72 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java @@ -1,53 +1,80 @@ package org.vitrivr.cineast.api.rest.resolvers; -import java.io.File; -import java.io.IOException; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + public class FileSystemObjectResolver implements ObjectResolver, AutoCloseable { - private final MediaObjectReader lookup; - private final File baseDir; + private final MediaObjectReader lookup; + private final File baseDir; - private final ObjectToFileResolver object2File; + private final LoadingCache descriptorLoadingCache; - public FileSystemObjectResolver(File basedir, MediaObjectReader lookup) { - this.lookup = lookup; - this.baseDir = basedir; - object2File = (dir, obj) -> new File(dir, obj.getPath()); - } + private final ObjectToFileResolver object2File; - public FileSystemObjectResolver(File basedir, MediaObjectReader lookup, ObjectToFileResolver transform) { - this.lookup = lookup; - this.baseDir = basedir; - object2File = transform; - } + private static final Logger LOGGER = LogManager.getLogger(); + public FileSystemObjectResolver(File basedir, MediaObjectReader lookup) { + this(basedir, lookup, (dir, obj) -> new File(dir, obj.getPath())); + } - @Override - public ResolutionResult resolve(String id) { + public FileSystemObjectResolver(File basedir, MediaObjectReader lookup, ObjectToFileResolver transform) { + this.lookup = lookup; + this.baseDir = basedir; + this.object2File = transform; - if (id == null) { - return null; + this.descriptorLoadingCache = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build( + new CacheLoader<>() { + @Override + public MediaObjectDescriptor load(String key) { + return lookup.lookUpObjectById(key); + } + } + ); } - MediaObjectDescriptor descriptor = this.lookup.lookUpObjectById(id); - if (!descriptor.exists()) { - return null; - } + @Override + public ResolutionResult resolve(String id) { + + if (id == null) { + return null; + } + + MediaObjectDescriptor descriptor; + try { + descriptor = descriptorLoadingCache.get(id); + } catch (ExecutionException e) { + LOGGER.error(e); + return null; + } - try { - return new ResolutionResult(object2File.resolve(baseDir, descriptor)); - } catch (IOException e) { - e.printStackTrace(); - return null; + if (!descriptor.exists()) { + return null; + } + + try { + return new ResolutionResult(object2File.resolve(baseDir, descriptor)); + } catch (IOException e) { + LOGGER.error(e); + return null; + } } - } - @Override - public void close() { - this.lookup.close(); - } + @Override + public void close() { + this.lookup.close(); + } } From 7e5f44689de683e5f4d1596b097904acfb93e1ff Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Fri, 18 Mar 2022 15:42:37 +0100 Subject: [PATCH 40/72] Added MergingPoseDetector --- .../vitrivr/cineast/core/data/Skeleton.java | 9 ++ .../cineast/core/features/SkeletonPose.java | 6 +- .../core/util/pose/MergingPoseDetector.java | 149 ++++++++++++++++++ .../util/pose/MovenetMultiposeDetector.java | 6 +- .../core/util/pose/OpenPoseDetector.java | 4 +- .../cineast/core/util/pose/PoseDetector.java | 12 ++ 6 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/PoseDetector.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java index 057ee3558..0033fa817 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -73,6 +73,10 @@ public Skeleton( } + public Skeleton() { + //empty skeleton + } + public List getPoints() { ArrayList _return = new ArrayList<>(POINT_COUNT); for (int i = 0; i < POINT_COUNT; ++i) { @@ -97,5 +101,10 @@ public float getWeight(SkeletonPointName name) { return this.weights[name.ordinal()]; } + public void setPointWeighted(SkeletonPointName pointName, float weight, Point2D_F32 point) { + this.weights[pointName.ordinal()] = weight; + this.points[pointName.ordinal()] = point; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index af7552280..5aae6f927 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -26,7 +26,9 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.HungarianAlgorithm; +import org.vitrivr.cineast.core.util.pose.MergingPoseDetector; import org.vitrivr.cineast.core.util.pose.OpenPoseDetector; +import org.vitrivr.cineast.core.util.pose.PoseDetector; import org.vitrivr.cottontail.client.SimpleClient; import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; @@ -50,7 +52,7 @@ public class SkeletonPose extends AbstractFeatureModule { private static final String PERSON_ID_COL = "person"; private static final String FEATURE_COL = "skeleton"; private static final String WEIGHT_COL = "weights"; - private OpenPoseDetector detector = new OpenPoseDetector(); + private PoseDetector detector = new MergingPoseDetector(); public SkeletonPose() { super("feature_skeletonpose", (float) (16 * Math.PI), 12); @@ -85,7 +87,7 @@ protected void persist(Collection> skeletons) { } private synchronized List detectSkeletons(MultiImage img) { - return detector.recognizePoses(img.getBufferedImage()); + return detector.detectPoses(img.getBufferedImage()); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java new file mode 100644 index 000000000..019500678 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java @@ -0,0 +1,149 @@ +package org.vitrivr.cineast.core.util.pose; + +import georegression.struct.point.Point2D_F32; +import org.vitrivr.cineast.core.data.Skeleton; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class MergingPoseDetector implements PoseDetector{ + + private final PoseDetector pd1, pd2; + + private static final float weightThreshold = 0.25f; + + public MergingPoseDetector(PoseDetector pd1, PoseDetector pd2) { + this.pd1 = pd1; + this.pd2 = pd2; + } + + public MergingPoseDetector() { + this(new OpenPoseDetector(), new MovenetMultiposeDetector()); + } + + private static boolean matches(Skeleton skeleton1, Skeleton skeleton2) { + + float distanceThreshold = 0.2f * Math.max(size(skeleton1), size(skeleton2)); + distanceThreshold *= distanceThreshold; + + int matchCount = 0; + int mismatchCount = 0; + + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + + if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { + + Point2D_F32 p1 = skeleton1.getPoint(point); + Point2D_F32 p2 = skeleton2.getPoint(point); + + if (p1.distance2(p2) > distanceThreshold) { + ++mismatchCount; + } else { + ++matchCount; + } + } + } + + return matchCount >= 3 && matchCount >= 2 * mismatchCount; + + } + + private static float size(Skeleton skeleton) { + + float xmin = Float.MAX_VALUE, ymin = Float.MAX_VALUE, xmax = 0, ymax = 0; + + int pointCount = 0; + + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + + if (skeleton.getWeight(point) >= weightThreshold) { + Point2D_F32 p = skeleton.getPoint(point); + + xmin = Math.min(p.x, xmin); + ymin = Math.min(p.x, ymin); + xmax = Math.max(p.x, xmax); + ymax = Math.max(p.x, ymax); + + ++pointCount; + + } + } + + if (pointCount < 2) { + return 0f; + } + + return Math.max(xmax - xmin, ymax - ymin); + + } + + private static Skeleton merge(Skeleton skeleton1, Skeleton skeleton2) { + + Skeleton merged = new Skeleton(); + + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + + if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { + + float weight = Math.max(skeleton1.getWeight(point), skeleton2.getWeight(point)); + Point2D_F32 p = skeleton1.getPoint(point).plus(skeleton2.getPoint(point)).times(0.5f); + merged.setPointWeighted(point, weight, p); + + } else if (skeleton1.getWeight(point) >= weightThreshold) { + merged.setPointWeighted(point, skeleton1.getWeight(point), skeleton1.getPoint(point)); + } else if (skeleton2.getWeight(point) >= weightThreshold) { + merged.setPointWeighted(point, skeleton2.getWeight(point), skeleton2.getPoint(point)); + + } + } + + return merged; + + } + + private static List merge(List list1, List list2) { + + if (list1.isEmpty()) { + return list2; + } + + if (list2.isEmpty()) { + return list1; + } + + ArrayList merged = new ArrayList<>(); + + for (Skeleton skeleton1 : list1) { + Skeleton toAdd = skeleton1; + + Iterator iter = list2.iterator(); + + while (iter.hasNext()) { + Skeleton skeleton2 = iter.next(); + + if (matches(skeleton1, skeleton2)) { + toAdd = merge(skeleton1, skeleton2); + iter.remove(); + break; + } + } + + merged.add(toAdd); + } + + merged.addAll(list2); + + return merged; + + } + + @Override + public List detectPoses(BufferedImage img) { + return merge( + pd1.detectPoses(img), + pd2.detectPoses(img) + ); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java index b2fd21365..27b24ef15 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java @@ -12,7 +12,7 @@ import java.awt.*; import java.awt.image.BufferedImage; -import java.util.*; +import java.util.ArrayList; import java.util.List; @@ -20,7 +20,7 @@ * Detects up to 6 complete poses * see https://tfhub.dev/google/movenet/multipose/lightning/1 */ -public class MovenetMultiposeDetector { +public class MovenetMultiposeDetector implements PoseDetector { private static final String RESOURCE_PATH = "resources/movenet_multipose_lightning/"; private static final String FUNCTION = "serving_default"; @@ -34,7 +34,7 @@ public MovenetMultiposeDetector() { } - public List recognizePoses(BufferedImage img) { + public List detectPoses(BufferedImage img) { final int imageSize = 256; float scaling = ((float) imageSize) / Math.max(img.getWidth(), img.getHeight()); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java index 596ce7201..91dbae3e0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java @@ -24,7 +24,7 @@ /** * Based on https://github.com/liljom/openpose-tf-mobilenet-java */ -public class OpenPoseDetector implements AutoCloseable +public class OpenPoseDetector implements PoseDetector, AutoCloseable { private static final String MODEL_FILE = "resources/thin.pb"; private static final String INPUT_NAME = "image"; @@ -65,7 +65,7 @@ public void close() { this.graph.close(); } - public List recognizePoses(BufferedImage img) + public List detectPoses(BufferedImage img) { final int imageSize = 512; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/PoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/PoseDetector.java new file mode 100644 index 000000000..9dce491c8 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/PoseDetector.java @@ -0,0 +1,12 @@ +package org.vitrivr.cineast.core.util.pose; + +import org.vitrivr.cineast.core.data.Skeleton; + +import java.awt.image.BufferedImage; +import java.util.List; + +public interface PoseDetector { + + List detectPoses(BufferedImage img); + +} From 0314539dcd480978990417ff935d66d57fa20e33 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Mon, 28 Mar 2022 12:49:11 +0200 Subject: [PATCH 41/72] Updated Cottontail DB client dependency to 0.14.0-SNAPSHOT. --- .../cottontaildb/CottontailEntityCreator.java | 16 ++++----- .../db/cottontaildb/CottontailSelector.java | 34 +++++++++---------- .../db/cottontaildb/CottontailWriter.java | 4 +-- .../cli/db/LSC21TemporalUpdateCommand.java | 3 +- gradle.properties | 3 +- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 6fc97f424..f52171157 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -85,9 +85,9 @@ public boolean createTagEntity() { this.cottontail.client.create(create); /* tag ids should be unique */ - this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, IndexType.HASH_UQ, txId); + this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, IndexType.BTREE_UQ, txId); /* tag names do not necessarily have to be unique */ - this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.HASH, txId); + this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.BTREE, txId); /* could be used for autocomplete */ this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId); @@ -114,7 +114,7 @@ public boolean createMultiMediaObjectsEntity() { this.cottontail.client.create(entity); /* Create index. */ - this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); + this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -141,8 +141,8 @@ public boolean createSegmentEntity() { this.cottontail.client.create(entity); /* Create indexes. */ - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.HASH, txId); + this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); + this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -166,7 +166,7 @@ public boolean createMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); + this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -190,7 +190,7 @@ public boolean createSegmentMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); + this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -273,7 +273,7 @@ public boolean createHashNonUniqueIndex(String entityName, String column) { final long txId = this.cottontail.client.begin(); try { final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - this.createIndex(fqn, column, IndexType.HASH, txId); + this.createIndex(fqn, column, IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 543ac48f6..b3deb3497 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -23,20 +23,18 @@ import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Direction; import org.vitrivr.cottontail.client.language.basics.Distances; +import org.vitrivr.cottontail.client.language.basics.predicate.And; +import org.vitrivr.cottontail.client.language.basics.predicate.Expression; +import org.vitrivr.cottontail.client.language.basics.predicate.Or; +import org.vitrivr.cottontail.client.language.basics.predicate.Predicate; import org.vitrivr.cottontail.client.language.ddl.AboutEntity; import org.vitrivr.cottontail.client.language.dql.Query; -import org.vitrivr.cottontail.client.language.extensions.And; -import org.vitrivr.cottontail.client.language.extensions.Literal; -import org.vitrivr.cottontail.client.language.extensions.Or; -import org.vitrivr.cottontail.client.language.extensions.Predicate; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import static org.vitrivr.cineast.core.util.CineastConstants.*; - public final class CottontailSelector implements DBSelector { private static final Logger LOGGER = LogManager.getLogger(); @@ -100,7 +98,7 @@ public List> getNearestNeighbourRows(int k, f @Override public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv", queryConfig)); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv", queryConfig)); try { final TupleIterator results = this.cottontail.client.query(query); final List _return = new LinkedList<>(); @@ -117,7 +115,7 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v @Override public List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv-gen", qc)); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv-gen", qc)); try { return toSingleCol(this.cottontail.client.query(query), vectorName); } catch (StatusRuntimeException e) { @@ -158,7 +156,7 @@ public List> getRows(String fieldName, Relati } private List> getRowsHelper(String fieldName, String op, Object[] mapped, String dbQueryID) { - final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, op, mapped)).queryId(dbQueryID); + final Query query = new Query(this.fqn).select("*", null).where(new Expression(fieldName, op, mapped)).queryId(dbQueryID); try { return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { @@ -180,10 +178,10 @@ public List> getFulltextRows(int rows, String /* Process predicates. */ if (queryConfig != null && !queryConfig.getRelevantSegmentIds().isEmpty()) { final Set relevant = queryConfig.getRelevantSegmentIds(); - final Literal segmentIds = new Literal(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); - query.where(new And(segmentIds, new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0))); + final Expression segmentIds = new Expression(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); + query.where(new And(segmentIds, new Expression(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0))); } else { - query.where(new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0)); + query.where(new Expression(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0)); } try { @@ -209,14 +207,14 @@ public List> getRowsAND(List atomics = conditions.stream().map(c -> { final String op = toOperator(c.getMiddle()); - return new Literal(c.getLeft(), op, c.getRight().stream().map(PrimitiveTypeProvider::toObject).toArray()); + return new Expression(c.getLeft(), op, c.getRight().stream().map(PrimitiveTypeProvider::toObject).toArray()); }).collect(Collectors.toList()); /* */ final Optional predicates = atomics.stream().reduce(And::new); if (qc != null && !qc.getRelevantSegmentIds().isEmpty()) { final Set relevant = qc.getRelevantSegmentIds(); - final Literal segmentIds = new Literal(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); + final Expression segmentIds = new Expression(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); if (predicates.isPresent()) { query.where(new And(segmentIds, predicates.get())); } else { @@ -246,7 +244,7 @@ public List> getMetadataBySpec(List> getMetadataByIdAndSpec(List ids, List spec, String idColName, String dbQueryID) { final Query query = new Query(this.fqn).select("*", null).queryId(dbQueryID == null ? "md-id-spec" : dbQueryID); final Optional predicates = generateQueryFromMetadataSpec(spec); - final Literal segmentIds = new Literal(idColName, "IN", ids.toArray()); + final Expression segmentIds = new Expression(idColName, "IN", ids.toArray()); if (predicates.isPresent()) { query.where(new And(segmentIds, predicates.get())); } else { @@ -260,10 +258,10 @@ public Optional generateQueryFromMetadataSpec(List> atomics = spec.stream().map(s -> { List singleSpecPredicates = new ArrayList<>(); if (!s.domain.isEmpty() && !s.domain.equals("*")) { - singleSpecPredicates.add(new Literal(DOMAIN_COL_NAME, "=", s.domain)); + singleSpecPredicates.add(new Expression(DOMAIN_COL_NAME, "=", s.domain)); } if (!s.key.isEmpty() && !s.key.equals("*")) { - singleSpecPredicates.add(new Literal(KEY_COL_NAME, "=", s.key)); + singleSpecPredicates.add(new Expression(KEY_COL_NAME, "=", s.key)); } return singleSpecPredicates.stream().reduce(And::new); }).collect(Collectors.toList()); @@ -418,7 +416,7 @@ private Query knn(int k, float[] vector, String column, ReadableQueryConfig conf /* Add relevant segments (optional). */ if (!relevant.isEmpty()) { - query.where(new Literal(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray())); + query.where(new Expression(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray())); } return query; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index 9a9b84768..ec7646fad 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -7,10 +7,10 @@ import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; +import org.vitrivr.cottontail.client.language.basics.predicate.Expression; import org.vitrivr.cottontail.client.language.dml.BatchInsert; import org.vitrivr.cottontail.client.language.dml.Insert; import org.vitrivr.cottontail.client.language.dql.Query; -import org.vitrivr.cottontail.client.language.extensions.Literal; public final class CottontailWriter extends AbstractPersistencyWriter { @@ -45,7 +45,7 @@ public void close() { /* No op */ } @Override public boolean exists(String key, String value) { - final Query query = new Query(this.fqn).exists().where(new Literal(key, "=", value)); + final Query query = new Query(this.fqn).exists().where(new Expression(key, "=", value)); final TupleIterator results = this.cottontail.client.query(query); final Boolean b = results.next().asBoolean("exists"); if (b != null) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java index b361e0c00..952949ac3 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java @@ -19,6 +19,7 @@ import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; +import org.vitrivr.cottontail.client.language.basics.predicate.Expression; import org.vitrivr.cottontail.client.language.dml.Update; import org.vitrivr.cottontail.client.language.dql.Query; import org.vitrivr.cottontail.grpc.CottontailGrpc; @@ -132,7 +133,7 @@ public void run() { resetAbsTime ? 0.0 : sAbs), new Pair<>(MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME, resetAbsTime ? 0.0 : (sAbs + 1)) - ).where(new org.vitrivr.cottontail.client.language.extensions.Literal(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())).txId(txId); + ).where(new Expression(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())).txId(txId); updates.add(update); } catch (Exception e) { LOGGER.warn("Could not update " + segment.getSegmentId() + " due to exception: " + e.getMessage()); diff --git a/gradle.properties b/gradle.properties index be686a949..678e7de4a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,8 @@ version_commonstext=1.9 version_commonsio=2.8.0 version_commonslang3=3.12.0 version_commonsmath3=3.6.1 -version_cottontaildb_proto=0.13.0 +version_cottontaildb_proto=0.14.0-SNAPSHOT + version_ffmpeg=5.0-1.5.7 version_gson=2.8.6 version_guava=30.1.1-jre From 04120217c814a2909ecbe08093e910e18bbdaac6 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Mon, 28 Mar 2022 12:50:31 +0200 Subject: [PATCH 42/72] Name of entity for SkeletonPose is now consistent with the other Cineast entities. --- .../cineast/core/features/SkeletonPose.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 5aae6f927..5b0a542ba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -55,7 +55,7 @@ public class SkeletonPose extends AbstractFeatureModule { private PoseDetector detector = new MergingPoseDetector(); public SkeletonPose() { - super("feature_skeletonpose", (float) (16 * Math.PI), 12); + super("features_skeletonpose", (float) (16 * Math.PI), 12); } @Override @@ -67,9 +67,9 @@ public void init(PersistencyWriterSupplier phandlerSupply) { @Override public void initalizePersistentLayer(Supplier supply) { supply.get().createFeatureEntity(this.tableName, false, - new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), - new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), - new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) + new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), + new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), + new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) ); } @@ -87,7 +87,7 @@ protected void persist(Collection> skeletons) { } private synchronized List detectSkeletons(MultiImage img) { - return detector.detectPoses(img.getBufferedImage()); + return new ArrayList<>(); } @Override @@ -268,7 +268,7 @@ private static ColumnName columnName(String name) { } private static ProjectionElement projectionElement(String name) { - return ProjectionElement.newBuilder().setColumn(columnName(name)).build(); + return ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setColumn(columnName(name)).build()).build(); } private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { @@ -311,20 +311,20 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i ).build(); //weighted Manhattan-distance plus correction term for missing elements - ProjectionElement distanceFunction = ProjectionElement.newBuilder().setFunction(/* Distance function */ + ProjectionElement distanceFunction = ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setFunction(/* Distance function */ functionBuilder("add").addArguments( - Expression.newBuilder().setFunction(functionBuilder("manhattanw") - .addArguments( - Expression.newBuilder().setColumn(columnName(FEATURE_COL)) - ).addArguments( - expression(query) - ).addArguments( - vectorDifference - )) + Expression.newBuilder().setFunction(functionBuilder("manhattanw") + .addArguments( + Expression.newBuilder().setColumn(columnName(FEATURE_COL)) + ).addArguments( + expression(query) + ).addArguments( + vectorDifference + )) ).addArguments( correctionTerm ) - ).setAlias(columnName("distance")).build(); + )).setAlias(columnName("distance")).build(); return QueryMessage.newBuilder().setQuery( From 592696c3ea6a81ece853dd48a63b70089b0c9535 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 28 Mar 2022 14:23:31 +0200 Subject: [PATCH 43/72] Added external model dependencies for pose detectors --- .../java/org/vitrivr/cineast/core/features/SkeletonPose.java | 2 +- .../org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java | 2 +- externalFiles.csv | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 5b0a542ba..4272b4319 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -87,7 +87,7 @@ protected void persist(Collection> skeletons) { } private synchronized List detectSkeletons(MultiImage img) { - return new ArrayList<>(); + return detector.detectPoses(img.getBufferedImage()); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java index 91dbae3e0..5af335bd8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/OpenPoseDetector.java @@ -26,7 +26,7 @@ */ public class OpenPoseDetector implements PoseDetector, AutoCloseable { - private static final String MODEL_FILE = "resources/thin.pb"; + private static final String MODEL_FILE = "resources/openpose/pose.pb"; private static final String INPUT_NAME = "image"; private static final String OUTPUT_NAME = "Openpose/concat_stage7"; private static final int[][] COCO_PAIRS = {{1, 2}, {1, 5}, {2, 3}, {3, 4}, {5, 6}, {6, 7}, {1, 8}, {8, 9}, {9, 10}, {1, 11}, diff --git a/externalFiles.csv b/externalFiles.csv index 78f49b198..36f739677 100644 --- a/externalFiles.csv +++ b/externalFiles.csv @@ -14,3 +14,5 @@ http://data.vitrivr.org/VisualTextCoEmbedding/visual-co-embedding.tar.gz,resourc https://tfhub.dev/google/universal-sentence-encoder/4?tf-hub-format=compressed,resources/VisualTextCoEmbedding/universal-sentence-encoder_4.tar.gz http://data.vitrivr.org/VisualTextCoEmbedding/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.tar.gz,resources/VisualTextCoEmbedding/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.tar.gz http://data.vitrivr.org/clip/clip.tar.gz,resources/CLIP.tar.gz +https://tfhub.dev/google/movenet/multipose/lightning/1?tf-hub-format=compressed,resources/movenet_multipose_lightning.tar.gz +http://data.vitrivr.org/pose/pose.pb,resources/openpose/pose.pb \ No newline at end of file From 92a71b49477bbbe717cbd79880547c4e789d54c6 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 28 Mar 2022 14:28:42 +0200 Subject: [PATCH 44/72] Minor cleanup in SkeletonPose --- .../cineast/core/features/SkeletonPose.java | 184 ++++-------------- 1 file changed, 33 insertions(+), 151 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 4272b4319..881321dc3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -1,12 +1,8 @@ package org.vitrivr.cineast.core.features; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import georegression.struct.point.Point2D_F32; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TObjectDoubleHashMap; -import org.vitrivr.cineast.core.config.DatabaseConfig; -import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatArrayIterable; import org.vitrivr.cineast.core.data.Pair; @@ -18,16 +14,12 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.PersistentTuple; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; import org.vitrivr.cineast.core.util.HungarianAlgorithm; import org.vitrivr.cineast.core.util.pose.MergingPoseDetector; -import org.vitrivr.cineast.core.util.pose.OpenPoseDetector; import org.vitrivr.cineast.core.util.pose.PoseDetector; import org.vitrivr.cottontail.client.SimpleClient; import org.vitrivr.cottontail.client.iterators.Tuple; @@ -38,8 +30,6 @@ import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Direction; import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; -import java.io.File; -import java.io.IOException; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -52,7 +42,7 @@ public class SkeletonPose extends AbstractFeatureModule { private static final String PERSON_ID_COL = "person"; private static final String FEATURE_COL = "skeleton"; private static final String WEIGHT_COL = "weights"; - private PoseDetector detector = new MergingPoseDetector(); + private final PoseDetector detector = new MergingPoseDetector(); public SkeletonPose() { super("features_skeletonpose", (float) (16 * Math.PI), 12); @@ -67,20 +57,20 @@ public void init(PersistencyWriterSupplier phandlerSupply) { @Override public void initalizePersistentLayer(Supplier supply) { supply.get().createFeatureEntity(this.tableName, false, - new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), - new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), - new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) + new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), + new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), + new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) ); } - protected void persist(Collection> skeletons) { + public void persist(Collection> skeletons) { if (skeletons == null || skeletons.isEmpty()) { return; } final List tuples = new ArrayList<>(skeletons.size()); int i = 0; - for (Pair skeleton : skeletons) { - final Pair pair = getAnglesandWeights(skeleton.second); + for (Pair skeleton : skeletons) { + final Pair pair = getAnglesAndWeights(skeleton.second); tuples.add(this.phandler.generateTuple(skeleton.first, i++, pair.first, pair.second)); } this.phandler.persist(tuples); @@ -124,7 +114,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc //query all skeletons for (Skeleton skeleton : skeletons) { - Pair pair = getAnglesandWeights(skeleton); + Pair pair = getAnglesAndWeights(skeleton); TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); while (tuples.hasNext()) { @@ -204,7 +194,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc return results; } - private Pair getAnglesandWeights(Skeleton skeleton) { + private Pair getAnglesAndWeights(Skeleton skeleton) { float[] angles = new float[]{ angle(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), @@ -285,42 +275,42 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i functionBuilder("vmin").addArguments( Expression.newBuilder().setColumn(columnName(WEIGHT_COL)) ).addArguments( - expression(weights) + expression(weights) ) ).build(); //assigns maximum distance for each element specified in the query but not present in the feature Expression correctionTerm = Expression.newBuilder().setFunction( functionBuilder("mul") - .addArguments( //constant - expression((float) Math.PI) - ).addArguments( //sub-expression - Expression.newBuilder().setFunction( - Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") - ).addArguments( - expression(queryWeightSum) - ).addArguments( - Expression.newBuilder().setFunction( - functionBuilder("vsum") - .addArguments(vectorDifference) + .addArguments( //constant + expression((float) Math.PI) + ).addArguments( //sub-expression + Expression.newBuilder().setFunction( + Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") + ).addArguments( + expression(queryWeightSum) + ).addArguments( + Expression.newBuilder().setFunction( + functionBuilder("vsum") + .addArguments(vectorDifference) + ) ) ) ) - ) ).build(); //weighted Manhattan-distance plus correction term for missing elements ProjectionElement distanceFunction = ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setFunction(/* Distance function */ functionBuilder("add").addArguments( - Expression.newBuilder().setFunction(functionBuilder("manhattanw") - .addArguments( - Expression.newBuilder().setColumn(columnName(FEATURE_COL)) - ).addArguments( - expression(query) - ).addArguments( - vectorDifference - )) + Expression.newBuilder().setFunction(functionBuilder("manhattanw") + .addArguments( + Expression.newBuilder().setColumn(columnName(FEATURE_COL)) + ).addArguments( + expression(query) + ).addArguments( + vectorDifference + )) ).addArguments( correctionTerm ) @@ -337,9 +327,9 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i .addElements(projectionElement(PERSON_ID_COL)) .addElements(distanceFunction) ).setOrder( - Order.newBuilder() - .addComponents(Component.newBuilder().setColumn(columnName(DB_DISTANCE_VALUE_QUALIFIER)) - .setDirection(Direction.ASCENDING)).build() + Order.newBuilder() + .addComponents(Component.newBuilder().setColumn(columnName(DB_DISTANCE_VALUE_QUALIFIER)) + .setDirection(Direction.ASCENDING)).build() ).setLimit(limit)).build(); } @@ -359,112 +349,4 @@ private static float min(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skele return min(skeleton.getWeight(p1), skeleton.getWeight(p2), skeleton.getWeight(p3)); } - // TODO or FIXME: Remove - public static void main(String[] args) throws IOException { - - File baseDir = new File("../../Downloads/VBS2022/VBS2022"); - File[] folders = baseDir.listFiles(File::isDirectory); - - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference<>() { - }; - - DatabaseConfig config = new DatabaseConfig(); - config.setHost("localhost"); - config.setPort(1865); - - final CottontailWrapper ctWrapper = new CottontailWrapper("localhost", 1865); - - - SkeletonPose sp = new SkeletonPose(); - - int batchSize = 10000; - boolean insert = true; - if (insert) { - sp.initalizePersistentLayer(() -> new CottontailEntityCreator(ctWrapper)); - sp.init(() -> new CottontailWriter(ctWrapper, 100, true)); - final List> skeletons = new LinkedList<>(); - for (File folder : folders) { - for (File file : folder.listFiles(f -> f.getName().endsWith(".json"))) { - final String segmentId = "v_" + file.getName().replaceAll("shot", "").replaceAll("_RKF.json", ""); - for (SkeletonEntry e : mapper.readValue(file, typeRef)) { - skeletons.add(new Pair<>(segmentId, e.toSkeleton())); - } - } - if (skeletons.size() >= batchSize) { - sp.persist(skeletons); - System.out.println("Persisted " + skeletons.size() + " entries..."); - skeletons.clear(); - } - } - - /* Final persist. */ - if (skeletons.size() > 0) { - sp.persist(skeletons); - System.out.println("Persisted " + skeletons.size() + " entries..."); - } - } - - - sp.init(() -> new CottontailSelector(ctWrapper)); - - Skeleton skeleton = mapper.readValue(new File(baseDir, "00001/shot00001_10_RKF.json"), typeRef).get(0).toSkeleton(); - - SegmentContainer container = new SegmentContainer() { - @Override - public String getId() { - return null; - } - - @Override - public String getSuperId() { - return null; - } - - @Override - public void setId(String id) { - - } - - @Override - public void setSuperId(String id) { - - } - - @Override - public List getSkeletons() { - return Collections.singletonList(skeleton); - } - }; - - List results = sp.getSimilar(container, new QueryConfig(null).setResultsPerModule(100)); - - int i = 0; - - for (ScoreElement element : results) { - System.out.println(i++ + ": " + element); - } - - } - - static class SkeletonEntry { - public int person_id; - public List> pose_keypoints; - - public Skeleton toSkeleton() { - float[] weights = new float[17]; - float[] coordinates = new float[34]; - - for (int i = 0; i < 17; ++i) { - coordinates[2 * i] = pose_keypoints.get(i).get(0); - coordinates[2 * i + 1] = pose_keypoints.get(i).get(1); - weights[i] = pose_keypoints.get(i).get(2); - } - - return new Skeleton(coordinates, weights); - - } - - } - } From 0838fef96904c9efa7af7eff06670feb12380d86 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 28 Mar 2022 15:28:26 +0200 Subject: [PATCH 45/72] Removed hack from TemporalQuery that is no longer needed --- .../org/vitrivr/cineast/api/messages/query/TemporalQuery.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java index 9fab868ab..f1d794925 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java @@ -78,7 +78,4 @@ public MessageType getMessageType() { return MessageType.Q_TEMPORAL; } - public void setTimeDistances(List distances) { - //FIXME there is a timeDistances field in the request that should presumably not be there. This is just a hotfix to get the message to serialize again. - } } From ab8090c9ca41d3db49c6c336b4c9d897de5ee7cc Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 4 Apr 2022 16:54:06 +0200 Subject: [PATCH 46/72] resolving merge-error which prevented index-creation, applying formatting --- .../api/messages/query/TemporalQuery.java | 10 +- .../resolvers/FileSystemObjectResolver.java | 101 ++-- .../FileSystemThumbnailResolver.java | 10 +- .../vitrivr/cineast/core/data/Skeleton.java | 171 +++--- .../core/data/providers/SkeletonProvider.java | 9 +- .../SkeletonQueryTermContainer.java | 39 +- .../core/data/segments/SegmentContainer.java | 61 +- .../cottontaildb/CottontailEntityCreator.java | 91 +-- .../db/cottontaildb/CottontailSelector.java | 18 +- .../db/cottontaildb/CottontailWriter.java | 12 +- .../cineast/core/features/SkeletonPose.java | 523 +++++++++--------- .../core/util/pose/MergingPoseDetector.java | 178 +++--- .../util/pose/MovenetMultiposeDetector.java | 135 +++-- .../importer/handlers/DataImportHandler.java | 11 +- 14 files changed, 705 insertions(+), 664 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java index f1d794925..eda68b6fc 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/TemporalQuery.java @@ -2,11 +2,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import org.vitrivr.cineast.api.messages.interfaces.MessageType; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; -import java.util.List; - /** * This object represents a temporal-query message of temporal query version 2, i.e. a request for a temporally staged similarity-search. */ @@ -23,11 +22,7 @@ public class TemporalQuery extends Query { private final List metadataAccessSpec; @JsonCreator - public TemporalQuery( - @JsonProperty(value = "queries", required = true) List queries, - @JsonProperty(value = "config", required = false) TemporalQueryConfig config, - @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec - ) { + public TemporalQuery(@JsonProperty(value = "queries", required = true) List queries, @JsonProperty(value = "config", required = false) TemporalQueryConfig config, @JsonProperty(value = "metadataAccessSpec", required = false) List metadataAccessSpec) { super(config); this.queries = queries; this.metadataAccessSpec = metadataAccessSpec; @@ -77,5 +72,4 @@ public List getMetadataAccessSpec() { public MessageType getMessageType() { return MessageType.Q_TEMPORAL; } - } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java index f3e6ced72..41d01b2a3 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemObjectResolver.java @@ -3,78 +3,73 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; -import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; - import java.io.File; import java.io.IOException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; +import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; public class FileSystemObjectResolver implements ObjectResolver, AutoCloseable { - private final MediaObjectReader lookup; - private final File baseDir; + private final MediaObjectReader lookup; + private final File baseDir; - private final LoadingCache descriptorLoadingCache; + private final LoadingCache descriptorLoadingCache; - private final ObjectToFileResolver object2File; + private final ObjectToFileResolver object2File; - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); - public FileSystemObjectResolver(File basedir, MediaObjectReader lookup) { - this(basedir, lookup, (dir, obj) -> new File(dir, obj.getPath())); - } + public FileSystemObjectResolver(File basedir, MediaObjectReader lookup) { + this(basedir, lookup, (dir, obj) -> new File(dir, obj.getPath())); + } - public FileSystemObjectResolver(File basedir, MediaObjectReader lookup, ObjectToFileResolver transform) { - this.lookup = lookup; - this.baseDir = basedir; - this.object2File = transform; - - this.descriptorLoadingCache = CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.MINUTES) - .build( - new CacheLoader<>() { - @Override - public MediaObjectDescriptor load(String key) { - return lookup.lookUpObjectById(key); - } - } - ); - } + public FileSystemObjectResolver(File basedir, MediaObjectReader lookup, ObjectToFileResolver transform) { + this.lookup = lookup; + this.baseDir = basedir; + this.object2File = transform; + this.descriptorLoadingCache = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<>() { + @Override + public MediaObjectDescriptor load(String key) { + return lookup.lookUpObjectById(key); + } + }); + } - @Override - public ResolutionResult resolve(String id) { - if (id == null) { - return null; - } + @Override + public ResolutionResult resolve(String id) { - MediaObjectDescriptor descriptor; - try { - descriptor = descriptorLoadingCache.get(id); - } catch (ExecutionException e) { - LOGGER.error(e); - return null; - } + if (id == null) { + return null; + } - if (!descriptor.exists()) { - return null; - } + MediaObjectDescriptor descriptor; + try { + descriptor = descriptorLoadingCache.get(id); + } catch (ExecutionException e) { + LOGGER.error(e); + return null; + } - try { - return new ResolutionResult(object2File.resolve(baseDir, descriptor)); - } catch (IOException e) { - LOGGER.error(e); - return null; - } + if (!descriptor.exists()) { + return null; } - @Override - public void close() { - this.lookup.close(); + try { + return new ResolutionResult(object2File.resolve(baseDir, descriptor)); + } catch (IOException e) { + LOGGER.error(e); + return null; } + } + + @Override + public void close() { + this.lookup.close(); + } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java index ed1043429..4ebd724ef 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java @@ -2,7 +2,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FilenameFilter; public class FileSystemThumbnailResolver implements ThumbnailResolver { @@ -24,20 +23,13 @@ public ResolutionResult resolve(String segmentId) { return null; } - //String fileName = segmentId.substring(0, segmentId.lastIndexOf("_")); - File dir = new File(this.baseFolder, split[1]); if (!dir.exists() || !dir.isDirectory()) { return null; } - File[] candidates = dir.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("shot" + split[1] + "_" + split[2]); - } - }); + File[] candidates = dir.listFiles((dir1, name) -> name.startsWith("shot" + split[1] + "_" + split[2])); if (candidates.length == 0) { return null; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java index 0033fa817..50c5d57ec 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/Skeleton.java @@ -3,108 +3,107 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import georegression.struct.point.Point2D_F32; - import java.util.ArrayList; import java.util.List; public class Skeleton { - public enum SkeletonPointName { - NOSE, - LEFT_EYE, - RIGHT_EYE, - LEFT_EAR, - RIGHT_EAR, - LEFT_SHOULDER, - RIGHT_SHOULDER, - LEFT_ELBOW, - RIGHT_ELBOW, - LEFT_WRIST, - RIGHT_WRIST, - LEFT_HIP, - RIGHT_HIP, - LEFT_KNEE, - RIGHT_KNEE, - LEFT_ANKLE, - RIGHT_ANKLE + public enum SkeletonPointName { + NOSE, + LEFT_EYE, + RIGHT_EYE, + LEFT_EAR, + RIGHT_EAR, + LEFT_SHOULDER, + RIGHT_SHOULDER, + LEFT_ELBOW, + RIGHT_ELBOW, + LEFT_WRIST, + RIGHT_WRIST, + LEFT_HIP, + RIGHT_HIP, + LEFT_KNEE, + RIGHT_KNEE, + LEFT_ANKLE, + RIGHT_ANKLE + } + + public static final SkeletonPointName[][] BONES = { + {SkeletonPointName.LEFT_ANKLE, SkeletonPointName.LEFT_KNEE}, + {SkeletonPointName.LEFT_KNEE, SkeletonPointName.LEFT_HIP}, + {SkeletonPointName.RIGHT_ANKLE, SkeletonPointName.RIGHT_KNEE}, + {SkeletonPointName.RIGHT_KNEE, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_HIP, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_HIP}, + {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_HIP}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.RIGHT_SHOULDER}, + {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_ELBOW}, + {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_ELBOW}, + {SkeletonPointName.LEFT_ELBOW, SkeletonPointName.LEFT_WRIST}, + {SkeletonPointName.RIGHT_ELBOW, SkeletonPointName.RIGHT_WRIST}, + {SkeletonPointName.LEFT_EYE, SkeletonPointName.RIGHT_EYE}, + {SkeletonPointName.NOSE, SkeletonPointName.LEFT_EYE}, + {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_EYE}, + {SkeletonPointName.LEFT_EYE, SkeletonPointName.LEFT_EAR}, + {SkeletonPointName.RIGHT_EYE, SkeletonPointName.RIGHT_EAR}, + {SkeletonPointName.NOSE, SkeletonPointName.LEFT_SHOULDER}, + {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_SHOULDER} + }; + + private static final int POINT_COUNT = 17; + + private final Point2D_F32[] points = new Point2D_F32[POINT_COUNT]; + private final float[] weights = new float[POINT_COUNT]; + + @JsonCreator + public Skeleton( + @JsonProperty("coordinates") float[] coordinates, + @JsonProperty("weights") float[] weights + ) { + + if (coordinates == null || coordinates.length < 2 * POINT_COUNT || weights == null || weights.length < POINT_COUNT) { + throw new IllegalArgumentException(); } - public static final SkeletonPointName[][] BONES = { - {SkeletonPointName.LEFT_ANKLE, SkeletonPointName.LEFT_KNEE}, - {SkeletonPointName.LEFT_KNEE, SkeletonPointName.LEFT_HIP}, - {SkeletonPointName.RIGHT_ANKLE, SkeletonPointName.RIGHT_KNEE}, - {SkeletonPointName.RIGHT_KNEE, SkeletonPointName.RIGHT_HIP}, - {SkeletonPointName.LEFT_HIP, SkeletonPointName.RIGHT_HIP}, - {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_HIP}, - {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_HIP}, - {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.RIGHT_SHOULDER}, - {SkeletonPointName.LEFT_SHOULDER, SkeletonPointName.LEFT_ELBOW}, - {SkeletonPointName.RIGHT_SHOULDER, SkeletonPointName.RIGHT_ELBOW}, - {SkeletonPointName.LEFT_ELBOW, SkeletonPointName.LEFT_WRIST}, - {SkeletonPointName.RIGHT_ELBOW, SkeletonPointName.RIGHT_WRIST}, - {SkeletonPointName.LEFT_EYE, SkeletonPointName.RIGHT_EYE}, - {SkeletonPointName.NOSE, SkeletonPointName.LEFT_EYE}, - {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_EYE}, - {SkeletonPointName.LEFT_EYE, SkeletonPointName.LEFT_EAR}, - {SkeletonPointName.RIGHT_EYE, SkeletonPointName.RIGHT_EAR}, - {SkeletonPointName.NOSE, SkeletonPointName.LEFT_SHOULDER}, - {SkeletonPointName.NOSE, SkeletonPointName.RIGHT_SHOULDER} - }; - - private static final int POINT_COUNT = 17; - - private final Point2D_F32[] points = new Point2D_F32[POINT_COUNT]; - private final float[] weights = new float[POINT_COUNT]; - - @JsonCreator - public Skeleton( - @JsonProperty("coordinates") float[] coordinates, - @JsonProperty("weights") float[] weights - ) { - - if (coordinates == null || coordinates.length < 2 * POINT_COUNT || weights == null || weights.length < POINT_COUNT) { - throw new IllegalArgumentException(); - } - - for (int i = 0; i < POINT_COUNT; ++i) { - this.weights[i] = weights[i]; - this.points[i] = new Point2D_F32(coordinates[2 * i], coordinates[2 * i + 1]); - } - + for (int i = 0; i < POINT_COUNT; ++i) { + this.weights[i] = weights[i]; + this.points[i] = new Point2D_F32(coordinates[2 * i], coordinates[2 * i + 1]); } - public Skeleton() { - //empty skeleton - } + } - public List getPoints() { - ArrayList _return = new ArrayList<>(POINT_COUNT); - for (int i = 0; i < POINT_COUNT; ++i) { - _return.add(this.points[i].copy()); - } - return _return; - } + public Skeleton() { + //empty skeleton + } - public List getPointsScaled(float scaleX, float scaleY) { - ArrayList _return = new ArrayList<>(POINT_COUNT); - for (int i = 0; i < POINT_COUNT; ++i) { - _return.add(new Point2D_F32(points[i].x * scaleX, points[i].y * scaleY)); - } - return _return; + public List getPoints() { + ArrayList _return = new ArrayList<>(POINT_COUNT); + for (int i = 0; i < POINT_COUNT; ++i) { + _return.add(this.points[i].copy()); } + return _return; + } - public Point2D_F32 getPoint(SkeletonPointName name) { - return this.points[name.ordinal()].copy(); + public List getPointsScaled(float scaleX, float scaleY) { + ArrayList _return = new ArrayList<>(POINT_COUNT); + for (int i = 0; i < POINT_COUNT; ++i) { + _return.add(new Point2D_F32(points[i].x * scaleX, points[i].y * scaleY)); } + return _return; + } - public float getWeight(SkeletonPointName name) { - return this.weights[name.ordinal()]; - } + public Point2D_F32 getPoint(SkeletonPointName name) { + return this.points[name.ordinal()].copy(); + } - public void setPointWeighted(SkeletonPointName pointName, float weight, Point2D_F32 point) { - this.weights[pointName.ordinal()] = weight; - this.points[pointName.ordinal()] = point; - } + public float getWeight(SkeletonPointName name) { + return this.weights[name.ordinal()]; + } + + public void setPointWeighted(SkeletonPointName pointName, float weight, Point2D_F32 point) { + this.weights[pointName.ordinal()] = weight; + this.points[pointName.ordinal()] = point; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java index 40efd472b..ff1b1622f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/SkeletonProvider.java @@ -1,14 +1,13 @@ package org.vitrivr.cineast.core.data.providers; -import org.vitrivr.cineast.core.data.Skeleton; - import java.util.Collections; import java.util.List; +import org.vitrivr.cineast.core.data.Skeleton; public interface SkeletonProvider { - default List getSkeletons() { - return Collections.emptyList(); - } + default List getSkeletons() { + return Collections.emptyList(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java index db14d4684..61550446b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/SkeletonQueryTermContainer.java @@ -1,33 +1,32 @@ package org.vitrivr.cineast.core.data.query.containers; -import org.vitrivr.cineast.core.data.Skeleton; -import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; -import org.vitrivr.cineast.core.util.web.DataURLParser; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import org.vitrivr.cineast.core.data.Skeleton; +import org.vitrivr.cineast.core.util.json.JacksonJsonProvider; +import org.vitrivr.cineast.core.util.web.DataURLParser; -public class SkeletonQueryTermContainer extends AbstractQueryTermContainer{ +public class SkeletonQueryTermContainer extends AbstractQueryTermContainer { - private final List skeletons = new ArrayList<>(); + private final List skeletons = new ArrayList<>(); - public SkeletonQueryTermContainer(String data) { - final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); - final String converted = DataURLParser.dataURLtoString(data, "application/json"); - final Skeleton[] skeletons = jsonProvider.toObject(converted, Skeleton[].class); - if (skeletons != null) { - this.skeletons.addAll(Arrays.asList(skeletons)); - } + public SkeletonQueryTermContainer(String data) { + final JacksonJsonProvider jsonProvider = new JacksonJsonProvider(); + final String converted = DataURLParser.dataURLtoString(data, "application/json"); + final Skeleton[] skeletons = jsonProvider.toObject(converted, Skeleton[].class); + if (skeletons != null) { + this.skeletons.addAll(Arrays.asList(skeletons)); } + } - public SkeletonQueryTermContainer(Collection skeletons) { - this.skeletons.addAll(skeletons); - } + public SkeletonQueryTermContainer(Collection skeletons) { + this.skeletons.addAll(skeletons); + } - @Override - public List getSkeletons() { - return this.skeletons; - } + @Override + public List getSkeletons() { + return this.skeletons; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java index 54bac1e3b..b05e73118 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java @@ -1,6 +1,24 @@ package org.vitrivr.cineast.core.data.segments; -import org.vitrivr.cineast.core.data.providers.*; +import org.vitrivr.cineast.core.data.providers.AudioFrameProvider; +import org.vitrivr.cineast.core.data.providers.AudioSTFTProvider; +import org.vitrivr.cineast.core.data.providers.AvgImgProvider; +import org.vitrivr.cineast.core.data.providers.BooleanExpressionProvider; +import org.vitrivr.cineast.core.data.providers.DurationProvider; +import org.vitrivr.cineast.core.data.providers.FrameListProvider; +import org.vitrivr.cineast.core.data.providers.IdProvider; +import org.vitrivr.cineast.core.data.providers.InstantProvider; +import org.vitrivr.cineast.core.data.providers.LocationProvider; +import org.vitrivr.cineast.core.data.providers.MedianImgProvider; +import org.vitrivr.cineast.core.data.providers.MeshProvider; +import org.vitrivr.cineast.core.data.providers.MostRepresentativeFrameProvider; +import org.vitrivr.cineast.core.data.providers.PathProvider; +import org.vitrivr.cineast.core.data.providers.SemanticMapProvider; +import org.vitrivr.cineast.core.data.providers.SkeletonProvider; +import org.vitrivr.cineast.core.data.providers.SubtitleItemProvider; +import org.vitrivr.cineast.core.data.providers.TagProvider; +import org.vitrivr.cineast.core.data.providers.TextProvider; +import org.vitrivr.cineast.core.data.providers.VoxelGridProvider; import org.vitrivr.cineast.core.features.extractor.Extractor; import org.vitrivr.cineast.core.features.retriever.Retriever; @@ -10,23 +28,24 @@ * During the offline phase, it is passed to an {@link Extractor}, and during the online phase, it is passed to a {@link Retriever}. */ public interface SegmentContainer - extends IdProvider, - AvgImgProvider, - DurationProvider, - MedianImgProvider, - MostRepresentativeFrameProvider, - SubtitleItemProvider, - PathProvider, - TagProvider, - FrameListProvider, - AudioFrameProvider, - AudioSTFTProvider, - MeshProvider, - VoxelGridProvider, - LocationProvider, - InstantProvider, - TextProvider, - SemanticMapProvider, - BooleanExpressionProvider, - SkeletonProvider -{} + extends IdProvider, + AvgImgProvider, + DurationProvider, + MedianImgProvider, + MostRepresentativeFrameProvider, + SubtitleItemProvider, + PathProvider, + TagProvider, + FrameListProvider, + AudioFrameProvider, + AudioSTFTProvider, + MeshProvider, + VoxelGridProvider, + LocationProvider, + InstantProvider, + TextProvider, + SemanticMapProvider, + BooleanExpressionProvider, + SkeletonProvider { + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index f52171157..b49aa0621 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -6,6 +6,7 @@ import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import io.grpc.Status; import io.grpc.StatusRuntimeException; import java.util.Arrays; import java.util.HashMap; @@ -18,6 +19,7 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.db.setup.EntityDefinition.EntityDefinitionBuilder; import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; @@ -76,8 +78,8 @@ public boolean createTagEntity() { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + TagReader.TAG_ENTITY_NAME; - final CreateEntity create = new CreateEntity(entityName) + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + TagReader.TAG_ENTITY_NAME; + final CreateEntity create = new CreateEntity(fqn) .column(TagReader.TAG_ID_COLUMNNAME, Type.STRING, -1, false) .column(TagReader.TAG_NAME_COLUMNNAME, Type.STRING, -1, false) .column(TagReader.TAG_DESCRIPTION_COLUMNNAME, Type.STRING, -1, false) @@ -85,11 +87,11 @@ public boolean createTagEntity() { this.cottontail.client.create(create); /* tag ids should be unique */ - this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, IndexType.BTREE_UQ, txId); + this.createIndex(TagReader.TAG_ENTITY_NAME, TagReader.TAG_ID_COLUMNNAME, IndexType.BTREE_UQ, txId); /* tag names do not necessarily have to be unique */ - this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.BTREE, txId); + this.createIndex(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.BTREE, txId); /* could be used for autocomplete */ - this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId); + this.createIndex(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId); this.cottontail.client.commit(txId); return true; @@ -104,8 +106,8 @@ public boolean createMultiMediaObjectsEntity() { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + MediaObjectDescriptor.ENTITY; - final CreateEntity entity = new CreateEntity(entityName) + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + MediaObjectDescriptor.ENTITY; + final CreateEntity entity = new CreateEntity(fqn) .column(MediaObjectDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaObjectDescriptor.FIELDNAMES[1], Type.INTEGER, -1, false) .column(MediaObjectDescriptor.FIELDNAMES[2], Type.STRING, -1, false) @@ -114,7 +116,7 @@ public boolean createMultiMediaObjectsEntity() { this.cottontail.client.create(entity); /* Create index. */ - this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); + this.createIndex(MediaObjectDescriptor.ENTITY, MediaObjectDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -128,8 +130,8 @@ public boolean createSegmentEntity() { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + MediaSegmentDescriptor.ENTITY; - final CreateEntity entity = new CreateEntity(entityName) + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + MediaSegmentDescriptor.ENTITY; + final CreateEntity entity = new CreateEntity(fqn) .column(MediaSegmentDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaSegmentDescriptor.FIELDNAMES[1], Type.STRING, -1, false) .column(MediaSegmentDescriptor.FIELDNAMES[2], Type.INTEGER, -1, false) @@ -141,8 +143,8 @@ public boolean createSegmentEntity() { this.cottontail.client.create(entity); /* Create indexes. */ - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.BTREE, txId); + this.createIndex(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.BTREE_UQ, txId); + this.createIndex(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -156,8 +158,8 @@ public boolean createMetadataEntity(String tableName) { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; - final CreateEntity entity = new CreateEntity(entityName) + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; + final CreateEntity entity = new CreateEntity(fqn) .column(MediaObjectMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaObjectMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) .column(MediaObjectMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) @@ -166,7 +168,8 @@ public boolean createMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -180,8 +183,8 @@ public boolean createSegmentMetadataEntity(String tableName) { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; - final CreateEntity entity = new CreateEntity(entityName) + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; + final CreateEntity entity = new CreateEntity(fqn) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) @@ -190,7 +193,8 @@ public boolean createSegmentMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -214,7 +218,15 @@ public boolean createFeatureEntity(String featureEntityName, boolean unique, Att final HashMap hints = new HashMap<>(1); extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(featureEntityName, extended); + var ent = this.createEntity(featureEntityName, extended); + long txId = this.cottontail.client.begin(); + boolean success = this.createIndex(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.BTREE, txId); + if (success) { + this.cottontail.client.commit(txId); + } else { + this.cottontail.client.rollback(txId); + } + return ent; } @Override @@ -222,14 +234,20 @@ public boolean createIdEntity(String entityName, AttributeDefinition... attribut final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING); System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(entityName, extended); + var ent = this.createEntity(entityName, extended); + long txId = this.cottontail.client.begin(); + boolean success = this.createIndex(entityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.BTREE, txId); + if (success) { + this.cottontail.client.commit(txId); + } else { + this.cottontail.client.rollback(txId); + } + return ent; } @Override public boolean createEntity(String entityName, AttributeDefinition... attributes) { - return this.createEntity( - new org.vitrivr.cineast.core.db.setup.EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build() - ); + return this.createEntity(new EntityDefinitionBuilder(entityName).withAttributes(attributes).build()); } @Override @@ -237,8 +255,8 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(); - final CreateEntity entity = new CreateEntity(entityName).txId(txId); + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(); + final CreateEntity entity = new CreateEntity(fqn).txId(txId); for (AttributeDefinition attribute : def.getAttributes()) { int length = -1; if ((attribute.getType() == VECTOR || attribute.getType() == BITSET) && attribute.getLength() > 0) { @@ -251,13 +269,13 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d /* Create Index. */ for (AttributeDefinition attribute : def.getAttributes()) { if (attribute.getType() == TEXT) { - this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId); + this.createIndex(def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId); } // TODO (LS, 18.11.2020) Shouldn't we also have abstract indices in the db abstraction layer? final Optional hint = attribute.getHint(INDEX_HINT); if (hint.isPresent()) { IndexType idx = IndexType.valueOf(hint.get()); - this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), idx, txId); + this.createIndex(def.getEntityName(), attribute.getName(), idx, txId); } } this.cottontail.client.commit(txId); @@ -272,8 +290,7 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d public boolean createHashNonUniqueIndex(String entityName, String column) { final long txId = this.cottontail.client.begin(); try { - final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - this.createIndex(fqn, column, IndexType.BTREE, txId); + this.createIndex(entityName, column, IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -341,9 +358,19 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { } - private void createIndex(String entityName, String attribute, IndexType type, long txId) { - final String indexName = entityName + ".idx_" + attribute + "_" + type.name().toLowerCase(); + private boolean createIndex(String entityName, String attribute, IndexType type, long txId) { + var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; + final String indexName = fqn + ".idx_" + attribute + "_" + type.name().toLowerCase(); final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); - this.cottontail.client.create(index); + try { + this.cottontail.client.create(index); + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.ALREADY_EXISTS.getCode()) { + LOGGER.warn("Index {} was not created because it already exists", indexName); + return false; + } + throw e; + } + return true; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index b3deb3497..d7b668b7c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -8,6 +8,18 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; @@ -30,11 +42,6 @@ import org.vitrivr.cottontail.client.language.ddl.AboutEntity; import org.vitrivr.cottontail.client.language.dql.Query; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - public final class CottontailSelector implements DBSelector { private static final Logger LOGGER = LogManager.getLogger(); @@ -508,7 +515,6 @@ private static List handleNearestNeighbourRespons } - T e = DistanceElement.create(distanceElementClass, id, distance); result.add(e); } catch (NullPointerException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index ec7646fad..d8cd07e75 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -24,7 +24,9 @@ public final class CottontailWriter extends AbstractPersistencyWriter { */ private String fqn; - /** The batch size to use for INSERTS. */ + /** + * The batch size to use for INSERTS. + */ private final int batchSize; private final boolean useTransactions; @@ -60,12 +62,12 @@ public boolean persist(List tuples) { long start = System.currentTimeMillis(); int size = tuples.size(); long txId = 0L; - if(useTransactions){ + if (useTransactions) { txId = this.cottontail.client.begin(); } try { BatchInsert insert = new BatchInsert().into(this.fqn).columns(this.names); - if(useTransactions){ + if (useTransactions) { insert.txId(txId); } while (!tuples.isEmpty()) { @@ -88,14 +90,14 @@ public boolean persist(List tuples) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); this.cottontail.client.insert(insert); } - if(useTransactions){ + if (useTransactions) { this.cottontail.client.commit(txId); } long stop = System.currentTimeMillis(); LOGGER.trace("Completed insert of {} elements in {} ms", size, stop - start); return true; } catch (StatusRuntimeException e) { - if(useTransactions){ + if (useTransactions) { this.cottontail.client.rollback(txId); } return false; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 881321dc3..ff63d7715 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -1,8 +1,20 @@ package org.vitrivr.cineast.core.features; +import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + import georegression.struct.point.Point2D_F32; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TObjectDoubleHashMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.FloatArrayIterable; import org.vitrivr.cineast.core.data.Pair; @@ -25,328 +37,329 @@ import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.grpc.CottontailGrpc; -import org.vitrivr.cottontail.grpc.CottontailGrpc.*; +import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; +import org.vitrivr.cottontail.grpc.CottontailGrpc.EntityName; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Expression; +import org.vitrivr.cottontail.grpc.CottontailGrpc.From; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Function; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Order; import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Component; import org.vitrivr.cottontail.grpc.CottontailGrpc.Order.Direction; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection; import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.ProjectionElement; - -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; -import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Query; +import org.vitrivr.cottontail.grpc.CottontailGrpc.QueryMessage; +import org.vitrivr.cottontail.grpc.CottontailGrpc.Scan; +import org.vitrivr.cottontail.grpc.CottontailGrpc.SchemaName; public class SkeletonPose extends AbstractFeatureModule { - private static final String PERSON_ID_COL = "person"; - private static final String FEATURE_COL = "skeleton"; - private static final String WEIGHT_COL = "weights"; - private final PoseDetector detector = new MergingPoseDetector(); - - public SkeletonPose() { - super("features_skeletonpose", (float) (16 * Math.PI), 12); + private static final String PERSON_ID_COL = "person"; + private static final String FEATURE_COL = "skeleton"; + private static final String WEIGHT_COL = "weights"; + private final PoseDetector detector = new MergingPoseDetector(); + + public SkeletonPose() { + super("features_skeletonpose", (float) (16 * Math.PI), 12); + } + + @Override + public void init(PersistencyWriterSupplier phandlerSupply) { + super.init(phandlerSupply); + this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, PERSON_ID_COL, FEATURE_COL, WEIGHT_COL); + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(this.tableName, false, + new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), + new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), + new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) + ); + } + + public void persist(Collection> skeletons) { + if (skeletons == null || skeletons.isEmpty()) { + return; } - - @Override - public void init(PersistencyWriterSupplier phandlerSupply) { - super.init(phandlerSupply); - this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, PERSON_ID_COL, FEATURE_COL, WEIGHT_COL); + final List tuples = new ArrayList<>(skeletons.size()); + int i = 0; + for (Pair skeleton : skeletons) { + final Pair pair = getAnglesAndWeights(skeleton.second); + tuples.add(this.phandler.generateTuple(skeleton.first, i++, pair.first, pair.second)); } + this.phandler.persist(tuples); + } - @Override - public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity(this.tableName, false, - new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), - new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), - new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) - ); - } + private synchronized List detectSkeletons(MultiImage img) { + return detector.detectPoses(img.getBufferedImage()); + } - public void persist(Collection> skeletons) { - if (skeletons == null || skeletons.isEmpty()) { - return; - } - final List tuples = new ArrayList<>(skeletons.size()); - int i = 0; - for (Pair skeleton : skeletons) { - final Pair pair = getAnglesAndWeights(skeleton.second); - tuples.add(this.phandler.generateTuple(skeleton.first, i++, pair.first, pair.second)); - } - this.phandler.persist(tuples); - } + @Override + public void processSegment(SegmentContainer segmentContainer) { - private synchronized List detectSkeletons(MultiImage img) { - return detector.detectPoses(img.getBufferedImage()); + VideoFrame representativeFrame = segmentContainer.getMostRepresentativeFrame(); + + if (representativeFrame == VideoFrame.EMPTY_VIDEO_FRAME) { + return; } - @Override - public void processSegment(SegmentContainer segmentContainer) { + this.persist(detectSkeletons(representativeFrame.getImage()).stream().map(it -> new Pair<>(segmentContainer.getId(), it)).collect(Collectors.toList())); + } - VideoFrame representativeFrame = segmentContainer.getMostRepresentativeFrame(); + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - if (representativeFrame == VideoFrame.EMPTY_VIDEO_FRAME) { - return; - } + List skeletons = sc.getSkeletons(); - this.persist(detectSkeletons(representativeFrame.getImage()).stream().map(it -> new Pair<>(segmentContainer.getId(), it)).collect(Collectors.toList())); + if (skeletons == null || skeletons.isEmpty()) { + return Collections.emptyList(); } - @Override - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + if (!(this.selector instanceof CottontailSelector)) { + return Collections.emptyList(); + } - List skeletons = sc.getSkeletons(); + SimpleClient client = ((CottontailSelector) this.selector).getWrapper().client; - if (skeletons == null || skeletons.isEmpty()) { - return Collections.emptyList(); - } + HashMap>> segmentDistancesMap = new HashMap<>(qc.getRawResultsPerModule() * skeletons.size()); - if (!(this.selector instanceof CottontailSelector)) { - return Collections.emptyList(); - } + int queryPersonId = 0; - SimpleClient client = ((CottontailSelector) this.selector).getWrapper().client; + //query all skeletons + for (Skeleton skeleton : skeletons) { - HashMap>> segmentDistancesMap = new HashMap<>(qc.getRawResultsPerModule() * skeletons.size()); + Pair pair = getAnglesAndWeights(skeleton); + TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); - int queryPersonId = 0; + while (tuples.hasNext()) { + Tuple tuple = tuples.next(); - //query all skeletons - for (Skeleton skeleton : skeletons) { + String segment = tuple.asString(GENERIC_ID_COLUMN_QUALIFIER); - Pair pair = getAnglesAndWeights(skeleton); - TupleIterator tuples = client.query(buildQuery(pair.first, pair.second, qc.getRawResultsPerModule())); + if (!segmentDistancesMap.containsKey(segment)) { + segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); + } - while (tuples.hasNext()) { - Tuple tuple = tuples.next(); + segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asFloat(DB_DISTANCE_VALUE_QUALIFIER)); - String segment = tuple.asString(GENERIC_ID_COLUMN_QUALIFIER); + } - if (!segmentDistancesMap.containsKey(segment)) { - segmentDistancesMap.put(segment, new TObjectDoubleHashMap<>()); - } + ++queryPersonId; - segmentDistancesMap.get(segment).put(new Pair<>(queryPersonId, tuple.asInt(PERSON_ID_COL)), tuple.asFloat(DB_DISTANCE_VALUE_QUALIFIER)); + } - } + ArrayList results = new ArrayList<>(segmentDistancesMap.size()); + + //compute assignment + if (queryPersonId == 1) { //only one query skeleton + for (String segment : segmentDistancesMap.keySet()) { + TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); + double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); + results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist))); + } + results.sort(SegmentScoreElement.SCORE_COMPARATOR.reversed()); + return results.subList(0, Math.min(results.size(), qc.getRawResultsPerModule()) - 1); + } - ++queryPersonId; + //more than query skeleton + for (String segment : segmentDistancesMap.keySet()) { - } + TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); - ArrayList results = new ArrayList<>(segmentDistancesMap.size()); - - //compute assignment - if (queryPersonId == 1) { //only one query skeleton - for (String segment : segmentDistancesMap.keySet()) { - TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); - double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); - results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist))); - } - results.sort(SegmentScoreElement.SCORE_COMPARATOR.reversed()); - return results.subList(0, Math.min(results.size(), qc.getRawResultsPerModule()) - 1); - } + if (distances.isEmpty()) { + continue; //should never happen + } + Set personIds = distances.keySet().stream().map(p -> p.second).collect(Collectors.toSet()); - //more than query skeleton - for (String segment : segmentDistancesMap.keySet()) { + if (personIds.size() == 1) { //only one retrieved skeleton + double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); + results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist) / skeletons.size())); + continue; + } - TObjectDoubleHashMap> distances = segmentDistancesMap.get(segment); + //more than one retrieved skeletons - if (distances.isEmpty()) { - continue; //should never happen - } + double[][] costs = new double[skeletons.size()][personIds.size()]; + TIntIntHashMap inversePersonIdMapping = new TIntIntHashMap(personIds.size()); + int i = 0; + for (int personId : personIds) { + inversePersonIdMapping.put(personId, i++); + } - Set personIds = distances.keySet().stream().map(p -> p.second).collect(Collectors.toSet()); + for (Pair p : distances.keySet()) { + costs[p.first][inversePersonIdMapping.get(p.second)] = -distances.get(p); + } - if (personIds.size() == 1) { //only one retrieved skeleton - double minDist = Arrays.stream(distances.values()).min().orElse(Double.MAX_VALUE); - results.add(new SegmentScoreElement(segment, this.correspondence.applyAsDouble(minDist) / skeletons.size())); - continue; - } + HungarianAlgorithm hungarianAlgorithm = new HungarianAlgorithm(costs); + int[] assignment = hungarianAlgorithm.execute(); - //more than one retrieved skeletons + double scoreSum = 0; - double[][] costs = new double[skeletons.size()][personIds.size()]; - TIntIntHashMap inversePersonIdMapping = new TIntIntHashMap(personIds.size()); - int i = 0; - for (int personId : personIds) { - inversePersonIdMapping.put(personId, i++); - } + for (i = 0; i < Math.min(personIds.size(), skeletons.size()); ++i) { + scoreSum += this.correspondence.applyAsDouble(-costs[i][assignment[i]]); + } - for (Pair p : distances.keySet()) { - costs[p.first][inversePersonIdMapping.get(p.second)] = -distances.get(p); - } + results.add(new SegmentScoreElement(segment, scoreSum / skeletons.size())); - HungarianAlgorithm hungarianAlgorithm = new HungarianAlgorithm(costs); - int[] assignment = hungarianAlgorithm.execute(); + } - double scoreSum = 0; + return results; + } - for (i = 0; i < Math.min(personIds.size(), skeletons.size()); ++i) { - scoreSum += this.correspondence.applyAsDouble(-costs[i][assignment[i]]); - } + private Pair getAnglesAndWeights(Skeleton skeleton) { + float[] angles = new float[]{ + angle(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), - results.add(new SegmentScoreElement(segment, scoreSum / skeletons.size())); + angle(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), - } + angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - return results; - } + angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), + angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) + }; - private Pair getAnglesAndWeights(Skeleton skeleton) { - float[] angles = new float[]{ - angle(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), - angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), - angle(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), - angle(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), - - angle(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), - angle(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), - angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), - - angle(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), - angle(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - - angle(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), - angle(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) - }; - - float[] weights = new float[]{ - min(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), - min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), - min(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), - min(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), - - min(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), - min(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), - min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), - - min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), - min(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - - min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), - min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) - }; - - for (int i = 0; i < weights.length; ++i) { - weights[i] = weights[i] >= 0.5f ? 1f : 0f; - } + float[] weights = new float[]{ + min(skeleton, Skeleton.SkeletonPointName.LEFT_ANKLE, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP), + min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP), + min(skeleton, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_ANKLE), - return new Pair<>(angles, weights); + min(skeleton, Skeleton.SkeletonPointName.LEFT_WRIST, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.LEFT_ELBOW, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.RIGHT_ELBOW, Skeleton.SkeletonPointName.RIGHT_WRIST), - } + min(skeleton, Skeleton.SkeletonPointName.LEFT_KNEE, Skeleton.SkeletonPointName.LEFT_HIP, Skeleton.SkeletonPointName.LEFT_SHOULDER), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_KNEE, Skeleton.SkeletonPointName.RIGHT_HIP, Skeleton.SkeletonPointName.RIGHT_SHOULDER), - private static Expression expression(float f) { - return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData(f)).build(); - } + min(skeleton, Skeleton.SkeletonPointName.LEFT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.LEFT_EAR), + min(skeleton, Skeleton.SkeletonPointName.RIGHT_SHOULDER, Skeleton.SkeletonPointName.NOSE, Skeleton.SkeletonPointName.RIGHT_EAR) + }; - private static Expression expression(float[] f) { - return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( - CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(f)) - ))).build(); + for (int i = 0; i < weights.length; ++i) { + weights[i] = weights[i] >= 0.5f ? 1f : 0f; } - private static Function.Builder functionBuilder(String name) { - return CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName(name)); - } + return new Pair<>(angles, weights); - private static ColumnName columnName(String name) { - return ColumnName.newBuilder().setName(name).build(); - } + } - private static ProjectionElement projectionElement(String name) { - return ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setColumn(columnName(name)).build()).build(); - } + private static Expression expression(float f) { + return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setFloatData(f)).build(); + } - private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { + private static Expression expression(float[] f) { + return Expression.newBuilder().setLiteral(CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector( + CottontailGrpc.FloatVector.newBuilder().addAllVector(new FloatArrayIterable(f)) + ))).build(); + } - float queryWeightSum = 0f; + private static Function.Builder functionBuilder(String name) { + return CottontailGrpc.Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName(name)); + } - for (float w : weights) { - queryWeightSum += w; - } + private static ColumnName columnName(String name) { + return ColumnName.newBuilder().setName(name).build(); + } - // element-wise difference between stored and provided weights - // counts how many elements are set in the query but are not set in the feature - Expression vectorDifference = Expression.newBuilder().setFunction(/* Nested, min() function */ - functionBuilder("vmin").addArguments( - Expression.newBuilder().setColumn(columnName(WEIGHT_COL)) - ).addArguments( - expression(weights) - ) - ).build(); - - //assigns maximum distance for each element specified in the query but not present in the feature - Expression correctionTerm = Expression.newBuilder().setFunction( - functionBuilder("mul") - .addArguments( //constant - expression((float) Math.PI) - ).addArguments( //sub-expression - Expression.newBuilder().setFunction( - Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") - ).addArguments( - expression(queryWeightSum) - ).addArguments( - Expression.newBuilder().setFunction( - functionBuilder("vsum") - .addArguments(vectorDifference) - ) - ) - ) - ) + private static ProjectionElement projectionElement(String name) { + return ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setColumn(columnName(name)).build()).build(); + } - ).build(); - - //weighted Manhattan-distance plus correction term for missing elements - ProjectionElement distanceFunction = ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setFunction(/* Distance function */ - functionBuilder("add").addArguments( - Expression.newBuilder().setFunction(functionBuilder("manhattanw") - .addArguments( - Expression.newBuilder().setColumn(columnName(FEATURE_COL)) - ).addArguments( - expression(query) - ).addArguments( - vectorDifference - )) - ).addArguments( - correctionTerm - ) - )).setAlias(columnName("distance")).build(); - - - return QueryMessage.newBuilder().setQuery( - Query.newBuilder().setFrom( - From.newBuilder().setScan(Scan.newBuilder().setEntity(EntityName.newBuilder() - .setName(this.tableName).setSchema(SchemaName.newBuilder().setName("cineast")))) - ).setProjection( - Projection.newBuilder() - .addElements(projectionElement(GENERIC_ID_COLUMN_QUALIFIER)) - .addElements(projectionElement(PERSON_ID_COL)) - .addElements(distanceFunction) - ).setOrder( - Order.newBuilder() - .addComponents(Component.newBuilder().setColumn(columnName(DB_DISTANCE_VALUE_QUALIFIER)) - .setDirection(Direction.ASCENDING)).build() - ).setLimit(limit)).build(); - } + private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, int limit) { - private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { - return (float) (Math.atan2(p2.y - c.y, p2.x - c.x) - Math.atan2(p1.y - c.y, p1.x - c.x)); - } + float queryWeightSum = 0f; - private static float angle(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { - return angle(skeleton.getPoint(p1), skeleton.getPoint(p2), skeleton.getPoint(p3)); + for (float w : weights) { + queryWeightSum += w; } - private static float min(float f, float g, float h) { - return Math.min(f, Math.min(g, h)); - } + // element-wise difference between stored and provided weights + // counts how many elements are set in the query but are not set in the feature + Expression vectorDifference = Expression.newBuilder().setFunction(/* Nested, min() function */ + functionBuilder("vmin").addArguments( + Expression.newBuilder().setColumn(columnName(WEIGHT_COL)) + ).addArguments( + expression(weights) + ) + ).build(); + + //assigns maximum distance for each element specified in the query but not present in the feature + Expression correctionTerm = Expression.newBuilder().setFunction( + functionBuilder("mul") + .addArguments( //constant + expression((float) Math.PI) + ).addArguments( //sub-expression + Expression.newBuilder().setFunction( + Function.newBuilder().setName(CottontailGrpc.FunctionName.newBuilder().setName("sub") + ).addArguments( + expression(queryWeightSum) + ).addArguments( + Expression.newBuilder().setFunction( + functionBuilder("vsum") + .addArguments(vectorDifference) + ) + ) + ) + ) - private static float min(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { - return min(skeleton.getWeight(p1), skeleton.getWeight(p2), skeleton.getWeight(p3)); - } + ).build(); + + //weighted Manhattan-distance plus correction term for missing elements + ProjectionElement distanceFunction = ProjectionElement.newBuilder().setExpression(Expression.newBuilder().setFunction(/* Distance function */ + functionBuilder("add").addArguments( + Expression.newBuilder().setFunction(functionBuilder("manhattanw") + .addArguments( + Expression.newBuilder().setColumn(columnName(FEATURE_COL)) + ).addArguments( + expression(query) + ).addArguments( + vectorDifference + )) + ).addArguments( + correctionTerm + ) + )).setAlias(columnName("distance")).build(); + + return QueryMessage.newBuilder().setQuery( + Query.newBuilder().setFrom( + From.newBuilder().setScan(Scan.newBuilder().setEntity(EntityName.newBuilder() + .setName(this.tableName).setSchema(SchemaName.newBuilder().setName("cineast")))) + ).setProjection( + Projection.newBuilder() + .addElements(projectionElement(GENERIC_ID_COLUMN_QUALIFIER)) + .addElements(projectionElement(PERSON_ID_COL)) + .addElements(distanceFunction) + ).setOrder( + Order.newBuilder() + .addComponents(Component.newBuilder().setColumn(columnName(DB_DISTANCE_VALUE_QUALIFIER)) + .setDirection(Direction.ASCENDING)).build() + ).setLimit(limit)).build(); + } + + private static float angle(Point2D_F32 p1, Point2D_F32 c, Point2D_F32 p2) { + return (float) (Math.atan2(p2.y - c.y, p2.x - c.x) - Math.atan2(p1.y - c.y, p1.x - c.x)); + } + + private static float angle(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { + return angle(skeleton.getPoint(p1), skeleton.getPoint(p2), skeleton.getPoint(p3)); + } + + private static float min(float f, float g, float h) { + return Math.min(f, Math.min(g, h)); + } + + private static float min(Skeleton skeleton, Skeleton.SkeletonPointName p1, Skeleton.SkeletonPointName p2, Skeleton.SkeletonPointName p3) { + return min(skeleton.getWeight(p1), skeleton.getWeight(p2), skeleton.getWeight(p3)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java index 019500678..627b6efb1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MergingPoseDetector.java @@ -1,149 +1,145 @@ package org.vitrivr.cineast.core.util.pose; import georegression.struct.point.Point2D_F32; -import org.vitrivr.cineast.core.data.Skeleton; - import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.vitrivr.cineast.core.data.Skeleton; -public class MergingPoseDetector implements PoseDetector{ +public class MergingPoseDetector implements PoseDetector { - private final PoseDetector pd1, pd2; + private final PoseDetector pd1, pd2; - private static final float weightThreshold = 0.25f; + private static final float weightThreshold = 0.25f; - public MergingPoseDetector(PoseDetector pd1, PoseDetector pd2) { - this.pd1 = pd1; - this.pd2 = pd2; - } + public MergingPoseDetector(PoseDetector pd1, PoseDetector pd2) { + this.pd1 = pd1; + this.pd2 = pd2; + } - public MergingPoseDetector() { - this(new OpenPoseDetector(), new MovenetMultiposeDetector()); - } + public MergingPoseDetector() { + this(new OpenPoseDetector(), new MovenetMultiposeDetector()); + } - private static boolean matches(Skeleton skeleton1, Skeleton skeleton2) { + private static boolean matches(Skeleton skeleton1, Skeleton skeleton2) { - float distanceThreshold = 0.2f * Math.max(size(skeleton1), size(skeleton2)); - distanceThreshold *= distanceThreshold; + float distanceThreshold = 0.2f * Math.max(size(skeleton1), size(skeleton2)); + distanceThreshold *= distanceThreshold; - int matchCount = 0; - int mismatchCount = 0; + int matchCount = 0; + int mismatchCount = 0; - for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { - if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { + if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { - Point2D_F32 p1 = skeleton1.getPoint(point); - Point2D_F32 p2 = skeleton2.getPoint(point); + Point2D_F32 p1 = skeleton1.getPoint(point); + Point2D_F32 p2 = skeleton2.getPoint(point); - if (p1.distance2(p2) > distanceThreshold) { - ++mismatchCount; - } else { - ++matchCount; - } - } + if (p1.distance2(p2) > distanceThreshold) { + ++mismatchCount; + } else { + ++matchCount; } - - return matchCount >= 3 && matchCount >= 2 * mismatchCount; - + } } - private static float size(Skeleton skeleton) { + return matchCount >= 3 && matchCount >= 2 * mismatchCount; - float xmin = Float.MAX_VALUE, ymin = Float.MAX_VALUE, xmax = 0, ymax = 0; + } - int pointCount = 0; + private static float size(Skeleton skeleton) { - for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + float xmin = Float.MAX_VALUE, ymin = Float.MAX_VALUE, xmax = 0, ymax = 0; - if (skeleton.getWeight(point) >= weightThreshold) { - Point2D_F32 p = skeleton.getPoint(point); + int pointCount = 0; - xmin = Math.min(p.x, xmin); - ymin = Math.min(p.x, ymin); - xmax = Math.max(p.x, xmax); - ymax = Math.max(p.x, ymax); + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { - ++pointCount; + if (skeleton.getWeight(point) >= weightThreshold) { + Point2D_F32 p = skeleton.getPoint(point); - } - } + xmin = Math.min(p.x, xmin); + ymin = Math.min(p.x, ymin); + xmax = Math.max(p.x, xmax); + ymax = Math.max(p.x, ymax); - if (pointCount < 2) { - return 0f; - } + ++pointCount; - return Math.max(xmax - xmin, ymax - ymin); + } + } + if (pointCount < 2) { + return 0f; } - private static Skeleton merge(Skeleton skeleton1, Skeleton skeleton2) { + return Math.max(xmax - xmin, ymax - ymin); - Skeleton merged = new Skeleton(); + } - for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { + private static Skeleton merge(Skeleton skeleton1, Skeleton skeleton2) { - if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { + Skeleton merged = new Skeleton(); - float weight = Math.max(skeleton1.getWeight(point), skeleton2.getWeight(point)); - Point2D_F32 p = skeleton1.getPoint(point).plus(skeleton2.getPoint(point)).times(0.5f); - merged.setPointWeighted(point, weight, p); + for (Skeleton.SkeletonPointName point : Skeleton.SkeletonPointName.values()) { - } else if (skeleton1.getWeight(point) >= weightThreshold) { - merged.setPointWeighted(point, skeleton1.getWeight(point), skeleton1.getPoint(point)); - } else if (skeleton2.getWeight(point) >= weightThreshold) { - merged.setPointWeighted(point, skeleton2.getWeight(point), skeleton2.getPoint(point)); + if (skeleton1.getWeight(point) >= weightThreshold && skeleton2.getWeight(point) >= weightThreshold) { - } - } + float weight = Math.max(skeleton1.getWeight(point), skeleton2.getWeight(point)); + Point2D_F32 p = skeleton1.getPoint(point).plus(skeleton2.getPoint(point)).times(0.5f); + merged.setPointWeighted(point, weight, p); - return merged; + } else if (skeleton1.getWeight(point) >= weightThreshold) { + merged.setPointWeighted(point, skeleton1.getWeight(point), skeleton1.getPoint(point)); + } else if (skeleton2.getWeight(point) >= weightThreshold) { + merged.setPointWeighted(point, skeleton2.getWeight(point), skeleton2.getPoint(point)); + } } - private static List merge(List list1, List list2) { + return merged; - if (list1.isEmpty()) { - return list2; - } + } - if (list2.isEmpty()) { - return list1; - } + private static List merge(List list1, List list2) { + + if (list1.isEmpty()) { + return list2; + } - ArrayList merged = new ArrayList<>(); + if (list2.isEmpty()) { + return list1; + } - for (Skeleton skeleton1 : list1) { - Skeleton toAdd = skeleton1; + ArrayList merged = new ArrayList<>(); - Iterator iter = list2.iterator(); + for (Skeleton skeleton1 : list1) { + Skeleton toAdd = skeleton1; - while (iter.hasNext()) { - Skeleton skeleton2 = iter.next(); + Iterator iter = list2.iterator(); - if (matches(skeleton1, skeleton2)) { - toAdd = merge(skeleton1, skeleton2); - iter.remove(); - break; - } - } + while (iter.hasNext()) { + Skeleton skeleton2 = iter.next(); - merged.add(toAdd); + if (matches(skeleton1, skeleton2)) { + toAdd = merge(skeleton1, skeleton2); + iter.remove(); + break; } + } - merged.addAll(list2); + merged.add(toAdd); + } - return merged; + merged.addAll(list2); - } + return merged; - @Override - public List detectPoses(BufferedImage img) { - return merge( - pd1.detectPoses(img), - pd2.detectPoses(img) - ); - } + } + + @Override + public List detectPoses(BufferedImage img) { + return merge(pd1.detectPoses(img), pd2.detectPoses(img)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java index 27b24ef15..22d285974 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/pose/MovenetMultiposeDetector.java @@ -1,5 +1,11 @@ package org.vitrivr.cineast.core.util.pose; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; import org.tensorflow.ConcreteFunction; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; @@ -10,101 +16,92 @@ import org.tensorflow.types.TInt32; import org.vitrivr.cineast.core.data.Skeleton; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.util.ArrayList; -import java.util.List; - /** - * Detects up to 6 complete poses - * see https://tfhub.dev/google/movenet/multipose/lightning/1 + * Detects up to 6 complete poses see https://tfhub.dev/google/movenet/multipose/lightning/1 */ public class MovenetMultiposeDetector implements PoseDetector { - private static final String RESOURCE_PATH = "resources/movenet_multipose_lightning/"; - private static final String FUNCTION = "serving_default"; + private static final String RESOURCE_PATH = "resources/movenet_multipose_lightning/"; + private static final String FUNCTION = "serving_default"; - private final SavedModelBundle multiPose; - private final ConcreteFunction function; + private final SavedModelBundle multiPose; + private final ConcreteFunction function; - public MovenetMultiposeDetector() { - this.multiPose = SavedModelBundle.load(RESOURCE_PATH); - this.function = this.multiPose.function(FUNCTION); - } + public MovenetMultiposeDetector() { + this.multiPose = SavedModelBundle.load(RESOURCE_PATH); + this.function = this.multiPose.function(FUNCTION); + } - public List detectPoses(BufferedImage img) { + public List detectPoses(BufferedImage img) { - final int imageSize = 256; - float scaling = ((float) imageSize) / Math.max(img.getWidth(), img.getHeight()); - int xOffset = (int) ((imageSize - (img.getWidth() * scaling)) / 2f); - int yOffset = (int) ((imageSize - (img.getHeight() * scaling)) / 2f); + final int imageSize = 256; + float scaling = ((float) imageSize) / Math.max(img.getWidth(), img.getHeight()); + int xOffset = (int) ((imageSize - (img.getWidth() * scaling)) / 2f); + int yOffset = (int) ((imageSize - (img.getHeight() * scaling)) / 2f); - BufferedImage resizedImg; - if (img.getWidth() == imageSize && img.getHeight() == imageSize) { - resizedImg = img; - } else { - resizedImg = new BufferedImage(imageSize, imageSize, BufferedImage.TYPE_INT_RGB); - Graphics2D g2 = resizedImg.createGraphics(); - g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, - RenderingHints.VALUE_INTERPOLATION_BILINEAR); - g2.setColor(Color.white); - g2.fillRect(0, 0, imageSize, imageSize); - g2.drawImage(img, xOffset, yOffset, (int) (img.getWidth() * scaling), (int) (img.getHeight() * scaling), null); - g2.dispose(); - } + BufferedImage resizedImg; + if (img.getWidth() == imageSize && img.getHeight() == imageSize) { + resizedImg = img; + } else { + resizedImg = new BufferedImage(imageSize, imageSize, BufferedImage.TYPE_INT_RGB); + Graphics2D g2 = resizedImg.createGraphics(); + g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g2.setColor(Color.white); + g2.fillRect(0, 0, imageSize, imageSize); + g2.drawImage(img, xOffset, yOffset, (int) (img.getWidth() * scaling), (int) (img.getHeight() * scaling), null); + g2.dispose(); + } - int[] colors = resizedImg.getRGB(0, 0, imageSize, imageSize, null, 0, imageSize); - int[] rgb = new int[imageSize * imageSize * 3]; + int[] colors = resizedImg.getRGB(0, 0, imageSize, imageSize, null, 0, imageSize); + int[] rgb = new int[imageSize * imageSize * 3]; - for (int i = 0; i < colors.length; ++i) { - int j = i * 3; - rgb[j] = (colors[i] >> 16) & 0xFF; // r - rgb[j + 1] = (colors[i] >> 8) & 0xFF; // g - rgb[j + 2] = (colors[i] & 0xFF); //b - } + for (int i = 0; i < colors.length; ++i) { + int j = i * 3; + rgb[j] = (colors[i] >> 16) & 0xFF; // r + rgb[j + 1] = (colors[i] >> 8) & 0xFF; // g + rgb[j + 2] = (colors[i] & 0xFF); //b + } - float[] points = new float[6 * 56]; + float[] points = new float[6 * 56]; - try (Tensor imageTensor = TInt32.tensorOf(Shape.of(1, imageSize, imageSize, 3), DataBuffers.of(rgb))) { - TFloat32 pointsTensor = (TFloat32) this.function.call(imageTensor); - FloatDataBuffer floatBuffer = DataBuffers.of(points); - pointsTensor.read(floatBuffer); - pointsTensor.close(); - } + try (Tensor imageTensor = TInt32.tensorOf(Shape.of(1, imageSize, imageSize, 3), DataBuffers.of(rgb))) { + TFloat32 pointsTensor = (TFloat32) this.function.call(imageTensor); + FloatDataBuffer floatBuffer = DataBuffers.of(points); + pointsTensor.read(floatBuffer); + pointsTensor.close(); + } - final int resultLength = 56; + final int resultLength = 56; - ArrayList skeletons = new ArrayList<>(6); + ArrayList skeletons = new ArrayList<>(6); - for (int pose = 0; pose < 6; ++pose) { + for (int pose = 0; pose < 6; ++pose) { - int offset = pose * resultLength; + int offset = pose * resultLength; - float score = points[offset + 55]; + float score = points[offset + 55]; - if (score < 0.5f) { - continue; - } + if (score < 0.5f) { + continue; + } - float[] coords = new float[17 * 2]; - float[] weights = new float[17]; + float[] coords = new float[17 * 2]; + float[] weights = new float[17]; - for (int i = 0; i < 17; ++i) { - coords[2 * i + 1] = ((points[offset + 3 * i] * imageSize) - yOffset) / scaling; - coords[2 * i] = ((points[offset + 3 * i + 1] * imageSize) - xOffset) / scaling; - weights[i] = points[offset + 3 * i + 1]; - } + for (int i = 0; i < 17; ++i) { + coords[2 * i + 1] = ((points[offset + 3 * i] * imageSize) - yOffset) / scaling; + coords[2 * i] = ((points[offset + 3 * i + 1] * imageSize) - xOffset) / scaling; + weights[i] = points[offset + 3 * i + 1]; + } - skeletons.add( - new Skeleton(coords, weights) - ); + skeletons.add(new Skeleton(coords, weights)); - } + } - return skeletons; + return skeletons; - } + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java index 70eedbc11..1e68e876a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java @@ -1,5 +1,12 @@ package org.vitrivr.cineast.standalone.importer.handlers; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.DataSource; @@ -14,10 +21,6 @@ import org.vitrivr.cineast.standalone.monitoring.ImportTaskMonitor; import org.vitrivr.cottontail.client.language.ddl.CreateEntity; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.concurrent.*; - public abstract class DataImportHandler { From 14b2ccb7cb8e75428b76eef950bf19934bf0aba3 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 4 Apr 2022 20:15:08 +0200 Subject: [PATCH 47/72] Made expected patterns in FileSystemThumbnailResolver explicit --- .../FileSystemThumbnailResolver.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java index 4ebd724ef..43c17075c 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java @@ -23,25 +23,28 @@ public ResolutionResult resolve(String segmentId) { return null; } - File dir = new File(this.baseFolder, split[1]); - - if (!dir.exists() || !dir.isDirectory()) { - return null; + File[] candidates = new File[]{ + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".png"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".png"), + }; + + for (File candidate : candidates) { + if (candidate.exists() && candidate.canRead()) { + try { + return new ResolutionResult(candidate); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } } - File[] candidates = dir.listFiles((dir1, name) -> name.startsWith("shot" + split[1] + "_" + split[2])); - - if (candidates.length == 0) { - return null; - } - - //TODO prioritize file endings - - try { - return new ResolutionResult(candidates[0]); - } catch (FileNotFoundException e) { - e.printStackTrace(); - return null; - } + return null; } } From 563f78ae71140dead573cce9f5fba6ae1fdfd0d9 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 6 Apr 2022 09:46:07 +0200 Subject: [PATCH 48/72] Added cache-control header to ResolvedContentRoute --- .../vitrivr/cineast/api/rest/routes/ResolvedContentRoute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/routes/ResolvedContentRoute.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/routes/ResolvedContentRoute.java index a40e13096..3c9d0eac3 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/routes/ResolvedContentRoute.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/routes/ResolvedContentRoute.java @@ -36,7 +36,7 @@ public void handle(Context ctx) throws Exception { return; } - ctx.contentType(rresult.mimeType); // Can be removed with the next javalin release + ctx.header("Cache-Control", "public, max-age=86400"); ctx.seekableStream(rresult.stream, rresult.mimeType); ctx.status(200); } From e22fba49f477f1d7fa43082c582357a8c9367c17 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 6 Apr 2022 17:05:38 +0200 Subject: [PATCH 49/72] logic to fetch objects given a limit and a skip for pagination in a user interface --- .../vitrivr/cineast/core/db/DBSelector.java | 52 +++++++-------- .../cineast/core/db/DBSelectorSupplier.java | 2 +- .../cineast/core/db/ImporterSelector.java | 65 +++++-------------- .../vitrivr/cineast/core/db/NoDBSelector.java | 5 -- .../db/cottontaildb/CottontailSelector.java | 10 +++ .../core/db/dao/reader/MediaObjectReader.java | 16 +++++ .../db/dao/reader/MediaSegmentReader.java | 24 +++---- .../cineast/core/db/DBIntegrationTest.java | 35 ++++++++-- 8 files changed, 103 insertions(+), 106 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index df5c00881..ee74a1619 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.compare.ObjectToStringComparator; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,17 +52,12 @@ default List getNearestNeighboursGeneric(int k, f * @param type of the {@link DistanceElement} * @return a list of elements with their distance */ - default List getNearestNeighboursGeneric(int k, - PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, - ReadableQueryConfig config) { - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { + default List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { //Default-implementation for backwards compatibility. - return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, - distanceElementClass, config); + return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, distanceElementClass, config); } - LogManager.getLogger().error("{} does not support other queries than float-arrays.", - this.getClass().getSimpleName()); + LogManager.getLogger().error("{} does not support other queries than float-arrays.", this.getClass().getSimpleName()); throw new UnsupportedOperationException(); } @@ -76,14 +72,12 @@ default List getNearestNeighboursGeneric(int k, * @param The type T of the resulting type of the {@link DistanceElement}. * @return List of results. */ - List getBatchedNearestNeighbours(int k, List vectors, - String column, Class distanceElementClass, List configs); + List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs); /** * In contrast to {@link #getNearestNeighboursGeneric(int, float[], String, Class, ReadableQueryConfig)}, this method returns all elements of a row */ - List> getNearestNeighbourRows(int k, float[] vector, - String column, ReadableQueryConfig config); + List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config); /** * SELECT 'vectorname' from entity where 'fieldname' = 'value' @@ -93,10 +87,8 @@ List> getNearestNeighbourRows(int k, float[] /** * for legacy support, takes the float[] method by default */ - default List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, - String vectorName, ReadableQueryConfig qc) { - return getFeatureVectors(fieldName, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new) - .collect(Collectors.toList()); + default List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { + return getFeatureVectors(fieldName, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new).collect(Collectors.toList()); } default List> getRows(String fieldName, PrimitiveTypeProvider value) { @@ -141,14 +133,12 @@ default List> getRows(String fieldName, List< * @param terms The query terms. Individual terms will be connected by a logical OR. * @return List of rows that math the fulltext search. */ - List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, - String... terms); + List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms); /** * {@link #getRows(String, RelationalOperator, Iterable)} */ - default List> getRows(String fieldName, RelationalOperator operator, - PrimitiveTypeProvider value) { + default List> getRows(String fieldName, RelationalOperator operator, PrimitiveTypeProvider value) { return getRows(fieldName, operator, Collections.singleton(value)); } @@ -162,8 +152,7 @@ default List> getRows(String fieldName, Relat * @param values The values the field should be compared to. * @return List of rows (one row is represented by one Map of the field ames and the data contained in the field). */ - List> getRows(String fieldName, RelationalOperator operator, - Iterable values); + List> getRows(String fieldName, RelationalOperator operator, Iterable values); /** * Performs a boolean lookup based on multiple conditions, linked with AND. Each element of the list specifies one of the conditions - left middle right, i.e. id IN (1, 5, 7) @@ -179,8 +168,7 @@ default List> getRowsAND(List ids = rows.stream().map(x -> x.get(identifier).getString()) - .collect(Collectors.toSet()); + Set ids = rows.stream().map(x -> x.get(identifier).getString()).collect(Collectors.toSet()); if (relevant.size() == 0) { rows.forEach(map -> relevant.put(map.get(identifier).getString(), map)); @@ -275,7 +263,9 @@ default List> getMetadataByIdAndSpec(List getAll(String column); + default List getAll(String column) { + return getAll().stream().map(el -> el.get(column)).collect(Collectors.toList()); + } /** * SELECT columns from the table. Be careful with large entities @@ -298,12 +288,20 @@ default List> getAll(List columns, in return collect; } + /** + * SELECT * FROM entity ORDER BY order ASC LIMIT limit SKIP skip + *
    + * skip is also sometimes called offset. This is horribly inefficient in the default implementation, as it serializes to string and then sorts. + */ + default List> getAll(String order, int skip, int limit) { + return getAll().stream().sorted((o1, o2) -> ObjectToStringComparator.INSTANCE.compare(o1.get(order), o2.get(order))).skip(skip).limit(limit).collect(Collectors.toList()); + } + /** * Get all rows from all tables */ List> getAll(); - boolean existsEntity(String name); /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java index b83ef1c59..2538d4e98 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java @@ -8,6 +8,6 @@ * Important: This class is required because of signature clashes in {@link org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule} due * to type erasure! */ -public interface DBSelectorSupplier extends Supplier{ +public interface DBSelectorSupplier extends Supplier { } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java index 2138c6f2b..13f34bf59 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java @@ -55,33 +55,23 @@ public void close() { } @Override - public List getNearestNeighboursGeneric(int k, - PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, - ReadableQueryConfig config) { + public List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { List> results; - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { results = getNearestNeighbourRows(k, queryProvider.getFloatArray(), column, config); } else { results = getNearestNeighbourRows(k, queryProvider, column, config); } - return results.stream() - .map(m -> DistanceElement.create( - distanceElementClass, m.get(GENERIC_ID_COLUMN_QUALIFIER).getString(), m.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble())) - .limit(k) - .collect(Collectors.toList()); + return results.stream().map(m -> DistanceElement.create(distanceElementClass, m.get(GENERIC_ID_COLUMN_QUALIFIER).getString(), m.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble())).limit(k).collect(Collectors.toList()); } /** * Full table scan. Don't do it for performance-intensive stuff. */ @SuppressWarnings("unchecked") - private List> getNearestNeighbourRows(int k, - PrimitiveTypeProvider queryProvider, String column, ReadableQueryConfig config) { - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { - return getNearestNeighbourRows(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), - column, config); + private List> getNearestNeighbourRows(int k, PrimitiveTypeProvider queryProvider, String column, ReadableQueryConfig config) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { + return getNearestNeighbourRows(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, config); } LOGGER.debug("Switching to non-float based lookup, reading from file"); Importer importer = newImporter(this.file); @@ -90,8 +80,7 @@ private List> getNearestNeighbourRows(int k, FixedSizePriorityQueue> knn; if (queryProvider.getType().equals(ProviderDataType.BITSET)) { distance = new BitSetHammingDistance(); - knn = FixedSizePriorityQueue - .create(k, new BitSetComparator(column, distance, queryProvider.getBitSet())); + knn = FixedSizePriorityQueue.create(k, new BitSetComparator(column, distance, queryProvider.getBitSet())); } else { throw new RuntimeException(queryProvider.getType().toString()); } @@ -103,8 +92,7 @@ private List> getNearestNeighbourRows(int k, } double d; if (queryProvider.getType().equals(ProviderDataType.BITSET)) { - d = distance - .applyAsDouble(queryProvider.getBitSet(), map.get(column).getBitSet()); + d = distance.applyAsDouble(queryProvider.getBitSet(), map.get(column).getBitSet()); map.put("distance", new FloatTypeProvider((float) d)); knn.add(map); } else { @@ -127,8 +115,7 @@ private List> getNearestNeighbourRows(int k, @Override - public List> getNearestNeighbourRows(int k, float[] vector, - String column, ReadableQueryConfig config) { + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { config = QueryConfig.clone(config); @@ -136,8 +123,7 @@ public List> getNearestNeighbourRows(int k, f FloatArrayDistance distance = FloatArrayDistance.fromQueryConfig(config); - FixedSizePriorityQueue> knn = FixedSizePriorityQueue - .create(k, new PrimitiveTypeMapDistanceComparator(column, vector, distance)); + FixedSizePriorityQueue> knn = FixedSizePriorityQueue.create(k, new PrimitiveTypeMapDistanceComparator(column, vector, distance)); HashSet relevant = null; if (config.hasRelevantSegmentIds()) { @@ -154,8 +140,7 @@ public List> getNearestNeighbourRows(int k, f if (relevant != null && !relevant.contains(map.get(GENERIC_ID_COLUMN_QUALIFIER))) { continue; } - double d = distance - .applyAsDouble(vector, PrimitiveTypeProvider.getSafeFloatArray(map.get(column))); + double d = distance.applyAsDouble(vector, PrimitiveTypeProvider.getSafeFloatArray(map.get(column))); map.put("distance", new FloatTypeProvider((float) d)); knn.add(map); } @@ -225,8 +210,7 @@ public List> getRows(String fieldName, Primit } @Override - public List> getRows(String fieldName, - Iterable values) { + public List> getRows(String fieldName, Iterable values) { if (values == null) { return new ArrayList<>(0); } @@ -242,21 +226,6 @@ public List> getRows(String fieldName, return this.getRows(fieldName, valueArr); } - @Override - public List getAll(String column) { - List _return = new ArrayList<>(); - - Importer importer = newImporter(this.file); - Map map; - while ((map = importer.readNextAsMap()) != null) { - PrimitiveTypeProvider p = map.get(column); - if (p != null) { - _return.add(p); - } - } - return _return; - } - @Override public List> getAll() { @@ -281,21 +250,17 @@ public boolean existsEntity(String name) { protected abstract String getFileExtension(); @Override - public List getBatchedNearestNeighbours(int k, - List vectors, String column, Class distanceElementClass, - List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { // TODO Auto-generated method stub return null; } @Override - public List> getRows(String fieldName, - RelationalOperator operator, Iterable values) { + public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { throw new IllegalStateException("Not implemented."); } - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, - String... terms) { + public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { throw new IllegalStateException("Not implemented."); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java index 7cc568a70..9dad2b9c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java @@ -46,11 +46,6 @@ public List> getRows(String fieldName, Iterab return new ArrayList<>(0); } - @Override - public List getAll(String column) { - return new ArrayList<>(0); - } - @Override public List> getAll() { return new ArrayList<>(0); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index a46d941e5..036fb1f47 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -355,6 +355,16 @@ public List> getAll() { } } + @Override + public List> getAll(String order, int skip, int limit) { + final Query query = new Query(this.fqn).select("*", null) + .queryId(generateQueryID("get-all-order-skip-limit"+this.fqn)) + .order(order, Direction.ASC) + .skip(skip) + .limit(limit); + return processResults(this.cottontail.client.query(query)); + } + @Override public boolean existsEntity(String name) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(name)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java index 02d3d2481..1722f911f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java @@ -129,4 +129,20 @@ public List getAllObjects() { return _return; } + /** + * SELECT * from mediaobjects ORDER BY id ASC LIMIT limit SKIP skip + * + * @param limit how many objects should be fetched + * @param skip how many objects should be skipped + * @return descriptors + */ + public List getAllObjects(int limit, int skip) { + List> all = selector.getAll(MediaObjectDescriptor.FIELDNAMES[0], limit, skip); + List _return = new ArrayList<>(all.size()); + for (Map map : all) { + _return.add(mapToDescriptor(map)); + } + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java index 2c80994da..9f35c053f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java @@ -32,8 +32,7 @@ public MediaSegmentReader(DBSelector dbSelector) { this.selector.open(MediaSegmentDescriptor.ENTITY); } - private static Optional propertiesToDescriptor( - Map properties) { + private static Optional propertiesToDescriptor(Map properties) { if (properties.containsKey(FIELDNAMES[0]) && properties.containsKey(FIELDNAMES[1]) @@ -60,8 +59,7 @@ private static Optional propertiesToDescriptor( } public Optional lookUpSegment(String segmentId) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[0], segmentId); + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[0], segmentId); return descriptors.findFirst(); } @@ -78,15 +76,12 @@ public Map lookUpSegments(Iterable segme } public List lookUpSegmentsOfObject(String objectId) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[1], objectId); + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[1], objectId); return descriptors.collect(Collectors.toList()); } - public ListMultimap lookUpSegmentsOfObjects( - Iterable objectIds) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[1], objectIds); + public ListMultimap lookUpSegmentsOfObjects(Iterable objectIds) { + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[1], objectIds); return Multimaps.index(descriptors.iterator(), MediaSegmentDescriptor::getObjectId); } @@ -100,18 +95,15 @@ public List lookUpSegmentsByNumberRange(String objectId, return all.stream().filter(it -> it.getSequenceNumber() >= lower && it.getSequenceNumber() <= upper).collect(Collectors.toList()); } - private Stream lookUpSegmentsByField( - String fieldName, String fieldValue) { + private Stream lookUpSegmentsByField(String fieldName, String fieldValue) { return lookUpSegmentsByField(fieldName, Collections.singletonList(fieldValue)); } - private Stream lookUpSegmentsByField( - String fieldName, Iterable fieldValues) { + private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues) { return lookUpSegmentsByField(fieldName, fieldValues, null); } - private Stream lookUpSegmentsByField( - String fieldName, Iterable fieldValues, String queryID) { + private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues, String queryID) { String dbQueryID = DBQueryIDGenerator.generateQueryID("seg-lookup", queryID); Set uniqueFieldValues = new HashSet<>(); fieldValues.forEach(el -> uniqueFieldValues.add(new StringTypeProvider(el))); diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 080835836..19b66a09e 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -11,6 +11,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.commons.lang3.RandomStringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -50,10 +51,10 @@ public abstract class DBIntegrationTest { protected EntityCreator ec; protected QueryConfig queryConfig; protected static final String ID_COL_NAME = "id"; - protected static final int VECTOR_ELEMENT_COUNT = 11; - protected static final int MAX_VECTOR_ID = 10; - protected static final int TEXT_ELEMENT_COUNT = 8; - protected static final int MAX_TEXT_ID = 7; + protected static final int VECTOR_ELEMENT_COUNT = 110; + protected static final int MAX_VECTOR_ID = VECTOR_ELEMENT_COUNT - 1; + protected static final int TEXT_ELEMENT_COUNT = 80; + protected static final int MAX_TEXT_ID = TEXT_ELEMENT_COUNT - 1; /** * This is not called "feature" by design as it avoid the storage-layers doing optimization by col name */ @@ -143,10 +144,10 @@ protected void fillVectorData() { vector[0] = i; vector[1] = 1; vector[2] = 0; - vectors.add(writer.generateTuple(String.valueOf(i), vector)); + vectors.add(writer.generateTuple(String.format("%05d", i), vector)); } /* We write a second vector with the same id in the db */ - vectors.add(writer.generateTuple(String.valueOf(0), new float[]{0, 0, 0})); + vectors.add(writer.generateTuple(String.format("%05d", 0), new float[]{0, 0, 0})); writer.persist(vectors); } @@ -237,6 +238,26 @@ void knnSearch() { Assertions.assertTrue(result.get(1).getSegmentId().equals("0") || result.get(2).getSegmentId().equals("0")); } + @Test + @DisplayName("Get all skip limit") + void getAllSkip() { + selector.open(testVectorTableName); + int limit = 10; + IntStream.range(0, 3).forEach(i -> { + List> all = selector.getAll(ID_COL_NAME, i, limit); + Assertions.assertEquals(all.size(), limit); + all.forEach(el -> { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) >= i - 1, "id " + el.get(ID_COL_NAME).getString() + " was smaller than " + i); + // we use <= for 0 because there are two elements with id 0 in the table + if (i == 0 || i == 1) { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) <= i + limit, "id " + el.get(ID_COL_NAME).getString() + " was larger than " + (i + limit)); + } else { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) < i + limit, "id " + el.get(ID_COL_NAME).getString() + " was larger than " + (i + limit)); + } + }); + }); + } + /** * TODO: Currently not supported in Cottontail DB v0.12.0. Re-activate, once support is back. */ @@ -340,7 +361,7 @@ public void testRetrievalSingleFuzzy() { /** * TODO: Fuzzy search on whole phrases is currently not supported. - * + *

    * Something like "hello world"~1 would need to be implemented as either hello~1 AND world~1, but that is not implemented in the DBSelector / cottontail. * The cottontail implementation in december 19 parses hello world~1 as hello .. world~1, which is not what we're looking for * Therefore, this test serves as a note that this functionality is lacking. From db31f71c9ab843f9ece8f39a91a19980c2fd9c9b Mon Sep 17 00:00:00 2001 From: silvanheller Date: Thu, 7 Apr 2022 08:59:05 +0200 Subject: [PATCH 50/72] implementing api endpoint, cleaning up constant usage --- .../org/vitrivr/cineast/api/APIEndpoint.java | 2 + .../mediaobject/FindObjectAllGetHandler.java | 14 +---- .../FindObjectByIdPostHandler.java | 8 +-- .../mediaobject/FindObjectGetHandler.java | 48 +++++++-------- .../FindObjectPaginationGetHandler.java | 61 +++++++++++++++++++ .../FindObjectMetadataByDomainGetHandler.java | 8 +-- ...FindObjectMetadataByDomainPostHandler.java | 5 +- .../FindObjectMetadataByKeyGetHandler.java | 2 +- .../FindObjectMetadataByKeyPostHandler.java | 2 +- ...bjectMetadataFullyQualifiedGetHandler.java | 16 +++-- .../FindObjectMetadataPostHandler.java | 3 +- .../actions/tag/FindTagsGetHandler.java | 6 +- .../cineast/api/util/APIConstants.java | 7 +++ 13 files changed, 117 insertions(+), 65 deletions(-) create mode 100644 cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index ffe73f9cf..9d7c6ca96 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -28,6 +28,7 @@ import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectAllGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectByIdPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectPaginationGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByDomainGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByDomainPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByKeyGetHandler; @@ -401,6 +402,7 @@ private void registerRestOperations() { new FindObjectAllGetHandler(), new FindObjectByIdPostHandler(), new FindObjectGetHandler(), + new FindObjectPaginationGetHandler(), /* Segments */ new FindSegmentByIdPostHandler(), new FindSegmentsByIdGetHandler(), diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java index 4d956bbb9..4455353f1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java @@ -3,28 +3,20 @@ import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; -import java.util.List; import org.vitrivr.cineast.api.messages.result.MediaObjectQueryResult; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; -import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; import org.vitrivr.cineast.standalone.config.Config; public class FindObjectAllGetHandler implements GetRestHandler { - - public static final String TYPE_NAME = "type"; - public static final String ROUTE = "find/objects/all/"; // The more honest route -// public static final String ROUTE = "find/objects/all/:"+TYPE_NAME; @Override public MediaObjectQueryResult doGet(Context ctx) { - // TODO :type is not being used - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List multimediaobjectIds = ol.getAllObjects(); - ol.close(); - return new MediaObjectQueryResult("", multimediaobjectIds); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + return new MediaObjectQueryResult("", ol.getAllObjects()); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java index 332dcaefc..d9cb0f9fc 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java @@ -19,14 +19,12 @@ public class FindObjectByIdPostHandler implements ParsingPostRestHandler parameters = ctx.pathParamMap(); if (context == null || context.getIds().length == 0) { return new MediaObjectQueryResult("", new ArrayList<>(0)); } - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final Map objects = ol.lookUpObjects(Arrays.asList(context.getIds())); - ol.close(); - return new MediaObjectQueryResult("", new ArrayList<>(objects.values())); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());) { + return new MediaObjectQueryResult("", new ArrayList<>(ol.lookUpObjects(Arrays.asList(context.getIds())).values())); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java index d04340d23..3f6abb258 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.api.rest.handlers.actions.mediaobject; +import static org.vitrivr.cineast.api.util.APIConstants.ATTRIBUTE_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.VALUE_NAME; + import com.google.common.collect.Lists; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -15,14 +18,10 @@ public class FindObjectGetHandler implements GetRestHandler { - public static final String ATTRIBUTE_NAME = "attribute"; - public static final String VALUE_NAME = "value"; - public static final String ROUTE = "find/object/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(FindObjectGetHandler.class); - @Override public MediaObjectQueryResult doGet(Context ctx) { final Map parameters = ctx.pathParamMap(); @@ -30,29 +29,27 @@ public MediaObjectQueryResult doGet(Context ctx) { final String attribute = parameters.get(ATTRIBUTE_NAME); final String value = parameters.get(VALUE_NAME); - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - MediaObjectDescriptor object = null; - - switch (attribute.toLowerCase()) { - case "id": { - object = ol.lookUpObjectById(value); - break; - } - case "name": { - object = ol.lookUpObjectByName(value); - break; - } - case "path": { - object = ol.lookUpObjectByPath(value); - break; - } - default: { - LOGGER.error("Unknown attribute '{}' in FindObjectByActionHandler", attribute); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + MediaObjectDescriptor object = null; + switch (attribute.toLowerCase()) { + case "id": { + object = ol.lookUpObjectById(value); + break; + } + case "name": { + object = ol.lookUpObjectByName(value); + break; + } + case "path": { + object = ol.lookUpObjectByPath(value); + break; + } + default: { + LOGGER.error("Unknown attribute '{}' in FindObjectByActionHandler", attribute); + } } + return new MediaObjectQueryResult("", Lists.newArrayList(object)); } - - ol.close(); - return new MediaObjectQueryResult("", Lists.newArrayList(object)); } @Override @@ -75,6 +72,7 @@ public OpenApiDocumentation docs() { op.addTagsItem("Object"); }) .pathParam(ATTRIBUTE_NAME, String.class, p -> p.description("The attribute type of the value. One of: id, name, path")) + .pathParam(VALUE_NAME, String.class, p -> p.description("The actual value that you want to filter for")) .json("200", outClass()); } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java new file mode 100644 index 000000000..d953a2eed --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -0,0 +1,61 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.mediaobject; + +import static org.vitrivr.cineast.api.util.APIConstants.LIMIT_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.SKIP_NAME; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.api.messages.result.MediaObjectQueryResult; +import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; +import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; +import org.vitrivr.cineast.standalone.config.Config; + +public class FindObjectPaginationGetHandler implements GetRestHandler { + + public static final String ROUTE = "find/object/all/{" + SKIP_NAME + "}/{" + LIMIT_NAME + "}"; + + private static final Logger LOGGER = LogManager.getLogger(FindObjectPaginationGetHandler.class); + + @Override + public MediaObjectQueryResult doGet(Context ctx) { + final Map parameters = ctx.pathParamMap(); + + final var skip = Integer.parseInt(parameters.get(SKIP_NAME)); + final var limit = Integer.parseInt(parameters.get(LIMIT_NAME)); + + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + return new MediaObjectQueryResult("", ol.getAllObjects(limit, skip)); + } + } + + @Override + public Class outClass() { + return MediaObjectQueryResult.class; + } + + @Override + public String route() { + return ROUTE; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Get a fixed amount of objects from the sorted list"); + op.description("Equivalent to calling SELECT * FROM multimediaobject ORDER BY objectid ASC LIMIT limit SKIP skip. Mostly used for pagination when wanting to retrieve all objects"); + op.operationId("findObjectsPagination"); + op.addTagsItem("Object"); + }) + .pathParam(LIMIT_NAME, Integer.class, p -> p.description("How many object at most should be fetched")) + .pathParam(SKIP_NAME, Integer.class, p -> p.description("How many objects should be skipped")) + .json("200", outClass()); + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java index 57f16827a..692ee6ae0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java @@ -1,7 +1,7 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.DOMAIN_NAME; import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.OBJECT_ID_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.DOMAIN_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -21,8 +21,7 @@ public class FindObjectMetadataByDomainGetHandler implements GetRestHandler { - public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME + "}/by/id/{" + APIConstants.ID_QUALIFIER - + "}"; + public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME + "}/by/id/{" + APIConstants.ID_QUALIFIER + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { @@ -30,8 +29,7 @@ public MediaObjectMetadataQueryResult doGet(Context ctx) { final String objectId = parameters.get(OBJECT_ID_NAME); final String domain = parameters.get(DOMAIN_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", - service.findByDomain(objectId, domain)); + return new MediaObjectMetadataQueryResult("", service.findByDomain(objectId, domain)); } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java index 0cf4c9188..b8f238d8e 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.DOMAIN_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.DOMAIN_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -33,8 +33,7 @@ public MediaObjectMetadataQueryResult performPost(IdList ids, Context ctx) { } final String domain = parameters.get(DOMAIN_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", - service.findByDomain(ids.getIdList(), domain)); + return new MediaObjectMetadataQueryResult("", service.findByDomain(ids.getIdList(), domain)); } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java index b2d3cd4db..057ea76aa 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.KEY_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.KEY_NAME; import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.OBJECT_ID_NAME; import io.javalin.http.Context; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java index cdba7633f..89f9055f0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.KEY_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.KEY_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java index 9fc17ac39..d1526cc25 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java @@ -8,6 +8,7 @@ import org.vitrivr.cineast.api.rest.OpenApiCompatHelper; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; import org.vitrivr.cineast.api.rest.services.MetadataRetrievalService; +import org.vitrivr.cineast.api.util.APIConstants; /** * This class handles GET requests with an object id, domain and key and returns all matching metadata descriptors. @@ -20,20 +21,17 @@ public class FindObjectMetadataFullyQualifiedGetHandler implements GetRestHandler { public static final String OBJECT_ID_NAME = "id"; - public static final String DOMAIN_NAME = "domain"; - public static final String KEY_NAME = "key"; - public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + DOMAIN_NAME + "}/with/{" + KEY_NAME + "}"; + public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + APIConstants.DOMAIN_NAME + "}/with/{" + APIConstants.KEY_NAME + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { final Map parameters = ctx.pathParamMap(); final String objectId = parameters.get(OBJECT_ID_NAME); - final String domain = parameters.get(DOMAIN_NAME); - final String key = parameters.get(KEY_NAME); + final String domain = parameters.get(APIConstants.DOMAIN_NAME); + final String key = parameters.get(APIConstants.KEY_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", service.find(objectId, domain, key) - ); + return new MediaObjectMetadataQueryResult("", service.find(objectId, domain, key)); } public OpenApiDocumentation docs() { @@ -47,10 +45,10 @@ public OpenApiDocumentation docs() { .pathParam(OBJECT_ID_NAME, String.class, param -> { param.description("The object id"); }) - .pathParam(DOMAIN_NAME, String.class, param -> { + .pathParam(APIConstants.DOMAIN_NAME, String.class, param -> { param.description("The domain name"); }) - .pathParam(KEY_NAME, String.class, param -> param.description("Metadata key")) + .pathParam(APIConstants.KEY_NAME, String.class, param -> param.description("Metadata key")) .json("200", outClass()); } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java index 910f9a5f2..7c3b6745f 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java @@ -24,8 +24,7 @@ public MediaObjectMetadataQueryResult performPost(OptionallyFilteredIdList ids, return new MediaObjectMetadataQueryResult("", new ArrayList<>(0)); } final MetadataRetrievalService service = new MetadataRetrievalService(); - List descriptors = service - .lookupMultimediaMetadata(ids.getIdList()); + List descriptors = service.lookupMultimediaMetadata(ids.getIdList()); if (ids.hasFilters()) { final List filters = ids.getFilterList(); for (AbstractMetadataFilterDescriptor filter : filters) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java index f2314e1b0..8534a52d1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.api.rest.handlers.actions.tag; +import static org.vitrivr.cineast.api.util.APIConstants.ATTRIBUTE_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.VALUE_NAME; + import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; @@ -21,9 +24,6 @@ public class FindTagsGetHandler implements GetRestHandler { public static final String NAME_NAME = "name"; public static final String MATCHING_NAME = "matchingname"; - public static final String ATTRIBUTE_NAME = "attribute"; - public static final String VALUE_NAME = "value"; - public static final String ROUTE = "find/tags/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(FindTagsGetHandler.class); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java index d0b490991..35be383a9 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java @@ -11,6 +11,13 @@ public class APIConstants { public static final String CATEGORY_NAME = "category"; public static final String ENTITY_NAME = "entity"; public static final List ACCESS_ALL_METADATA; + public static final String DOMAIN_NAME = "domain"; + public static final String KEY_NAME = "key"; + public static final String VALUE_NAME = "value"; + public static final String ATTRIBUTE_NAME = "attribute"; + public static final String LIMIT_NAME = "limit"; + public static final String SKIP_NAME = "skip"; + static { List _spec = new ArrayList<>(); From 6da78f33a38bb1def0f02c9e97d7256a347ecb37 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Thu, 7 Apr 2022 14:20:21 +0200 Subject: [PATCH 51/72] wrapping up api implementation, adding count(*) command to rest api --- .../org/vitrivr/cineast/api/APIEndpoint.java | 2 + .../actions/bool/CountRowsGetHandler.java | 51 +++++++++++++++++++ .../FindObjectByIdPostHandler.java | 2 +- .../FindObjectPaginationGetHandler.java | 7 ++- .../cineast/api/util/APIConstants.java | 1 + .../data/entities/MediaObjectDescriptor.java | 2 +- .../vitrivr/cineast/core/db/DBSelector.java | 9 ++++ .../db/cottontaildb/CottontailSelector.java | 6 +++ .../db/dao/reader/AbstractEntityReader.java | 6 +++ .../core/db/dao/reader/MediaObjectReader.java | 6 +-- .../cineast/core/db/DBIntegrationTest.java | 15 ++++-- 11 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index 9d7c6ca96..83b6fc963 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -25,6 +25,7 @@ import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindSegmentFeaturesGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindSegmentTextGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindTagsForElementGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.bool.CountRowsGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectAllGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectByIdPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectGetHandler; @@ -429,6 +430,7 @@ private void registerRestOperations() { /* Boolean */ new FindDistinctElementsByColumnPostHandler(), new SelectFromTablePostHandler(), + new CountRowsGetHandler(), /* Status */ new StatusInvocationHandler() )); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java new file mode 100644 index 000000000..81c3f28ce --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java @@ -0,0 +1,51 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.bool; + +import static org.vitrivr.cineast.api.util.APIConstants.TABLE_NAME; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; +import org.vitrivr.cineast.standalone.config.Config; + +public class CountRowsGetHandler implements GetRestHandler { + + public static final String ROUTE = "count/table{" + TABLE_NAME + "}"; + + private static final Logger LOGGER = LogManager.getLogger(CountRowsGetHandler.class); + + @Override + public Integer doGet(Context ctx) { + try (final var selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get()) { + var tableName = ctx.pathParam(TABLE_NAME); + selector.open(tableName); + var count = selector.rowCount(); + LOGGER.trace("counted {} objects in table {}", count, tableName); + return count; + } + } + + @Override + public Class outClass() { + return Integer.class; + } + + @Override + public String route() { + return ROUTE; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Count objects"); + op.description("Equivalent to calling SELECT count(*) FROM table. Used to determined #pages for pagination in a frontend or statistical purposes"); + op.operationId("countRows"); + op.addTagsItem("Misc"); + }) + .json("200", outClass()); + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java index d9cb0f9fc..4651e2deb 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java @@ -22,7 +22,7 @@ public MediaObjectQueryResult performPost(IdList context, Context ctx) { if (context == null || context.getIds().length == 0) { return new MediaObjectQueryResult("", new ArrayList<>(0)); } - try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());) { + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { return new MediaObjectQueryResult("", new ArrayList<>(ol.lookUpObjects(Arrays.asList(context.getIds())).values())); } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java index d953a2eed..f42f17ce3 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -6,14 +6,11 @@ import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.api.messages.result.MediaObjectQueryResult; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; -import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; import org.vitrivr.cineast.standalone.config.Config; @@ -31,7 +28,9 @@ public MediaObjectQueryResult doGet(Context ctx) { final var limit = Integer.parseInt(parameters.get(LIMIT_NAME)); try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { - return new MediaObjectQueryResult("", ol.getAllObjects(limit, skip)); + var result = ol.getAllObjects(skip, limit); + LOGGER.trace("returning {} elements for limit {} and skip {}", result.size(), limit, skip); + return new MediaObjectQueryResult("", result); } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java index 35be383a9..92f7e317a 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java @@ -17,6 +17,7 @@ public class APIConstants { public static final String ATTRIBUTE_NAME = "attribute"; public static final String LIMIT_NAME = "limit"; public static final String SKIP_NAME = "skip"; + public static final String TABLE_NAME = "table"; static { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java index 2ff4f1d34..ad7657618 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java @@ -94,7 +94,7 @@ public MediaObjectDescriptor(@JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) String ob // Config.sharedConfig().getApi().getObjectLocation() + path; } - @JsonProperty + @JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) public final String getObjectId() { return objectId; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index ee74a1619..6cf9c661e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -20,6 +20,7 @@ import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.ProviderDataType; @@ -302,10 +303,18 @@ default List> getAll(String order, int skip, */ List> getAll(); + /** + * SELECT count(*) FROM table + */ + default Integer rowCount(){ + return getAll().size(); + } + boolean existsEntity(String name); /** * Healthcheck. Returns false if something is wrong */ boolean ping(); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 036fb1f47..3c5f3c6b7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -365,6 +365,12 @@ public List> getAll(String order, int skip, i return processResults(this.cottontail.client.query(query)); } + @Override + public Integer rowCount() { + final Query query = new Query(this.fqn).count().queryId("count-star-"+this.fqn); + return Math.toIntExact(this.cottontail.client.query(query).next().asLong(0)); + } + @Override public boolean existsEntity(String name) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(name)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java index dd4690480..e6e2e794a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java @@ -1,6 +1,8 @@ package org.vitrivr.cineast.core.db.dao.reader; import java.io.Closeable; +import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.DBSelector; @@ -27,4 +29,8 @@ public AbstractEntityReader(DBSelector selector) { public void close() { this.selector.close(); } + + public Integer rowCount(){ + return this.selector.rowCount(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java index 1722f911f..28981e03f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java @@ -132,12 +132,12 @@ public List getAllObjects() { /** * SELECT * from mediaobjects ORDER BY id ASC LIMIT limit SKIP skip * - * @param limit how many objects should be fetched * @param skip how many objects should be skipped + * @param limit how many objects should be fetched * @return descriptors */ - public List getAllObjects(int limit, int skip) { - List> all = selector.getAll(MediaObjectDescriptor.FIELDNAMES[0], limit, skip); + public List getAllObjects(int skip, int limit) { + List> all = selector.getAll(MediaObjectDescriptor.FIELDNAMES[0], skip, limit); List _return = new ArrayList<>(all.size()); for (Map map : all) { _return.add(mapToDescriptor(map)); diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 19b66a09e..cc1bfd330 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -53,7 +53,7 @@ public abstract class DBIntegrationTest { protected static final String ID_COL_NAME = "id"; protected static final int VECTOR_ELEMENT_COUNT = 110; protected static final int MAX_VECTOR_ID = VECTOR_ELEMENT_COUNT - 1; - protected static final int TEXT_ELEMENT_COUNT = 80; + protected static final int TEXT_ELEMENT_COUNT = 8; protected static final int MAX_TEXT_ID = TEXT_ELEMENT_COUNT - 1; /** * This is not called "feature" by design as it avoid the storage-layers doing optimization by col name @@ -196,14 +196,23 @@ void entitiesExist() { } @Test - @DisplayName("Verify element count") - void count() { + @DisplayName("Verify element count using getAll()") + void countGetAll() { selector.open(testVectorTableName); Assertions.assertEquals(VECTOR_ELEMENT_COUNT, selector.getAll().size()); selector.open(testTextTableName); Assertions.assertEquals(TEXT_ELEMENT_COUNT, selector.getAll().size()); } + @Test + @DisplayName("Verify element count using rowCount()") + void countRowCount() { + selector.open(testVectorTableName); + Assertions.assertEquals(VECTOR_ELEMENT_COUNT, selector.rowCount()); + selector.open(testTextTableName); + Assertions.assertEquals(TEXT_ELEMENT_COUNT, selector.rowCount()); + } + @Test @DisplayName("Verify elements exist by id") void entriesExistById() { From e0d69f8e15eab2126a66d6bb48f09596de99d626 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Thu, 7 Apr 2022 14:21:28 +0200 Subject: [PATCH 52/72] forcing correct serialization for frontends which are using the openapi generated code --- .../cineast/core/data/entities/MediaObjectDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java index 2ff4f1d34..ad7657618 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectDescriptor.java @@ -94,7 +94,7 @@ public MediaObjectDescriptor(@JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) String ob // Config.sharedConfig().getApi().getObjectLocation() + path; } - @JsonProperty + @JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) public final String getObjectId() { return objectId; } From f4b1e04aeb660c50eddc6b2285739610d8dd1196 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Thu, 7 Apr 2022 14:47:07 +0200 Subject: [PATCH 53/72] fixing query-id name --- .../cineast/core/db/cottontaildb/CottontailSelector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 3cf9e548b..937836c03 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -362,7 +362,7 @@ public List> getAll() { @Override public List> getAll(String order, int skip, int limit) { final Query query = new Query(this.fqn).select("*", null) - .queryId(generateQueryID("get-all-order-skip-limit"+this.fqn)) + .queryId(generateQueryID("get-all-order-skip-limit-"+this.fqn)) .order(order, Direction.ASC) .skip(skip) .limit(limit); From 87e198301bb776d0b4db2b3d4a75d92bd29bdace Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Fri, 8 Apr 2022 10:17:34 +0200 Subject: [PATCH 54/72] adding missing / to api endpoint --- .../api/rest/handlers/actions/bool/CountRowsGetHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java index 81c3f28ce..32a582abe 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java @@ -12,7 +12,7 @@ public class CountRowsGetHandler implements GetRestHandler { - public static final String ROUTE = "count/table{" + TABLE_NAME + "}"; + public static final String ROUTE = "count/table/{" + TABLE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(CountRowsGetHandler.class); From 49b73abf5b8f12f08f376409b3d3981dd683ea79 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Fri, 8 Apr 2022 10:43:51 +0200 Subject: [PATCH 55/72] consistently using try-with-resources for object, segment and metadata --- .../segment/FindSegmentByIdPostHandler.java | 8 ++--- .../segment/FindSegmentsByIdGetHandler.java | 16 ++++----- .../FindSegmentsByObjectIdGetHandler.java | 8 ++--- .../vitrivr/cineast/api/util/QueryUtil.java | 28 +++++++-------- .../cineast/standalone/cli/CliUtils.java | 36 ++++++++++--------- 5 files changed, 49 insertions(+), 47 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java index 3a131e312..4ed549965 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java @@ -23,10 +23,10 @@ public MediaSegmentQueryResult performPost(IdList ids, Context ctx) { if (ids == null || ids.getIds().length == 0) { return new MediaSegmentQueryResult("", new ArrayList<>(0)); } - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final Map segments = sl.lookUpSegments(Arrays.asList(ids.getIds())); - sl.close(); - return new MediaSegmentQueryResult("", new ArrayList<>(segments.values())); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());) { + final Map segments = sl.lookUpSegments(Arrays.asList(ids.getIds())); + return new MediaSegmentQueryResult("", new ArrayList<>(segments.values())); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java index d7930c7ba..490e0afe1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java @@ -22,14 +22,14 @@ public class FindSegmentsByIdGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); final String segmentId = parameters.get(ID_NAME); - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List list = sl.lookUpSegment(segmentId).map(s -> { - final List segments = new ArrayList<>(1); - segments.add(s); - return segments; - }).orElse(new ArrayList<>(0)); - sl.close(); - return new MediaSegmentQueryResult("", list); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final List list = sl.lookUpSegment(segmentId).map(s -> { + final List segments = new ArrayList<>(1); + segments.add(s); + return segments; + }).orElse(new ArrayList<>(0)); + return new MediaSegmentQueryResult("", list); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java index 0528c4827..898231bd4 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java @@ -21,10 +21,10 @@ public class FindSegmentsByObjectIdGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); final String objectId = parameters.get(ID_NAME); - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List list = sl.lookUpSegmentsOfObject(objectId); - sl.close(); - return new MediaSegmentQueryResult("", list); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final List list = sl.lookUpSegmentsOfObject(objectId); + return new MediaSegmentQueryResult("", list); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java index 42df3f4c5..d71b206a9 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java @@ -116,20 +116,20 @@ public static List findSegmentsSimilarTemporal(ContinuousRetriev .collect(Collectors.toList()); // TODO: New MediaSegmentReader for every request like FindSegmentByIdPostHandler or one persistent on per endpoint like AbstractQueryMessageHandler? - final var segmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - - var segmentIds = stagedResults.stream().flatMap( - resultsMap -> resultsMap.values().stream().flatMap( - pairs -> pairs.stream().map(pair -> pair.key) - ) - ).distinct().collect(Collectors.toList()); - - var segmentDescriptors = segmentReader.lookUpSegments(segmentIds, config.getQueryId()); - var stagedQueryResults = stagedResults.stream().map( - resultsMap -> resultsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()) - ).collect(Collectors.toList()); - - return TemporalScoring.score(segmentDescriptors, stagedQueryResults, query.getTimeDistances(), query.getMaxLength()); + try (var segmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + var segmentIds = stagedResults.stream().flatMap( + resultsMap -> resultsMap.values().stream().flatMap( + pairs -> pairs.stream().map(pair -> pair.key) + ) + ).distinct().collect(Collectors.toList()); + + var segmentDescriptors = segmentReader.lookUpSegments(segmentIds, config.getQueryId()); + var stagedQueryResults = stagedResults.stream().map( + resultsMap -> resultsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()) + ).collect(Collectors.toList()); + + return TemporalScoring.score(segmentDescriptors, stagedQueryResults, query.getTimeDistances(), query.getMaxLength()); + } } /** diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java index afa020d11..1758a3dbd 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java @@ -29,31 +29,33 @@ public class CliUtils { public static void printInfoForObject(String objectId, DBSelector selector) { System.out.println("= Retrieving object information for " + objectId + " ="); - MediaObjectReader objectReader = new MediaObjectReader(selector); - System.out.println(objectReader.lookUpObjectById(objectId)); - + try (var objectReader = new MediaObjectReader(selector)) { + System.out.println(objectReader.lookUpObjectById(objectId)); + } System.out.println("= Retrieving object metadata for ="); - MediaObjectMetadataReader reader = new MediaObjectMetadataReader(selector); - List metadataDescriptors = reader.lookupMultimediaMetadata(objectId); - metadataDescriptors.forEach(System.out::println); + try (var reader = new MediaObjectMetadataReader(selector)) { + List metadataDescriptors = reader.lookupMultimediaMetadata(objectId); + metadataDescriptors.forEach(System.out::println); + } } public static void printInfoForSegment(String segmentId, DBSelector selector, String _filterCategory, boolean printObjInfo) { System.out.println("= Retrieving segment information for " + segmentId + "="); - MediaSegmentReader segmentReader = new MediaSegmentReader(selector); - Optional segmentDescriptor = segmentReader.lookUpSegment(segmentId); - segmentDescriptor.ifPresent(System.out::println); - - segmentDescriptor.ifPresent(descriptor -> { - if (printObjInfo) { - printInfoForObject(descriptor.getObjectId(), selector); - } - }); + try (var segmentReader = new MediaSegmentReader(selector)) { + Optional segmentDescriptor = segmentReader.lookUpSegment(segmentId); + segmentDescriptor.ifPresent(System.out::println); + segmentDescriptor.ifPresent(descriptor -> { + if (printObjInfo) { + printInfoForObject(descriptor.getObjectId(), selector); + } + }); + } System.out.println("= Retrieving segment metadata ="); - MediaSegmentMetadataReader reader = new MediaSegmentMetadataReader(selector); - reader.lookupMultimediaMetadata(segmentId).forEach(System.out::println); + try (var reader = new MediaSegmentMetadataReader(selector)) { + reader.lookupMultimediaMetadata(segmentId).forEach(System.out::println); + } System.out.println("Retrieving all columns for segment " + segmentId); RetrievalRuntimeConfig retrievalRuntimeConfig = Config.sharedConfig().getRetriever(); From 7014a4b93fd83638ae229b56428d484db3cecf7d Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Fri, 8 Apr 2022 10:48:53 +0200 Subject: [PATCH 56/72] updating openapi specs --- docs/openapi.json | 190 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 168 insertions(+), 22 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index a817fa4a6..4d154d793 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -370,6 +370,7 @@ }, { "name" : "value", "in" : "path", + "description" : "The actual value that you want to filter for", "required" : true, "schema" : { "type" : "string" @@ -389,6 +390,45 @@ } } }, + "/api/v1/find/object/all/{skip}/{limit}" : { + "get" : { + "tags" : [ "Object" ], + "summary" : "Get a fixed amount of objects from the sorted list", + "description" : "Equivalent to calling SELECT * FROM multimediaobject ORDER BY objectid ASC LIMIT limit SKIP skip. Mostly used for pagination when wanting to retrieve all objects", + "operationId" : "findObjectsPagination", + "parameters" : [ { + "name" : "skip", + "in" : "path", + "description" : "How many objects should be skipped", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int32" + } + }, { + "name" : "limit", + "in" : "path", + "description" : "How many object at most should be fetched", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int32" + } + } ], + "responses" : { + "200" : { + "description" : "OK", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/MediaObjectQueryResult" + } + } + } + } + } + } + }, "/api/v1/find/segments/by/id" : { "post" : { "tags" : [ "Segment" ], @@ -1030,6 +1070,35 @@ } } }, + "/api/v1/count/table/{table}" : { + "get" : { + "tags" : [ "Misc" ], + "summary" : "Count objects", + "description" : "Equivalent to calling SELECT count(*) FROM table. Used to determined #pages for pagination in a frontend or statistical purposes", + "operationId" : "countRows", + "parameters" : [ { + "name" : "table", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "OK", + "content" : { + "application/json" : { + "schema" : { + "type" : "integer", + "format" : "int32" + } + } + } + } + } + } + }, "/api/v1/status" : { "get" : { "tags" : [ "Status" ], @@ -1248,8 +1317,7 @@ "type" : "object", "properties" : { "objectid" : { - "type" : "string", - "writeOnly" : true + "type" : "string" }, "name" : { "type" : "string" @@ -1265,9 +1333,6 @@ "type" : "boolean", "writeOnly" : true }, - "objectId" : { - "type" : "string" - }, "contentURL" : { "type" : "string" } @@ -1351,8 +1416,7 @@ "type" : "object", "properties" : { "queryId" : { - "type" : "string", - "format" : "uuid" + "type" : "string" }, "hints" : { "uniqueItems" : true, @@ -1395,16 +1459,16 @@ "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, - "distanceIfEmpty" : { + "distanceWeightsIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "normIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "distanceWeightsIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, - "normIfEmpty" : { + "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "rawResultsPerModule" : { @@ -1418,7 +1482,7 @@ "properties" : { "type" : { "type" : "string", - "enum" : [ "IMAGE", "AUDIO", "MOTION", "MODEL3D", "LOCATION", "PARAMETERISED_LOCATION", "TIME", "TEXT", "TAG", "SEMANTIC", "ID", "BOOLEAN" ] + "enum" : [ "IMAGE", "AUDIO", "MODEL3D", "LOCATION", "PARAMETERISED_LOCATION", "TIME", "TEXT", "TAG", "SEMANTIC", "SKELETON", "ID", "BOOLEAN" ] }, "data" : { "type" : "string", @@ -1558,7 +1622,47 @@ } }, "config" : { - "$ref" : "#/components/schemas/QueryConfig" + "$ref" : "#/components/schemas/TemporalQueryConfig" + }, + "metadataAccessSpec" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/MetadataAccessSpecification" + } + }, + "maxLength" : { + "type" : "number", + "format" : "float" + }, + "messageType" : { + "type" : "string", + "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + }, + "timeDistances" : { + "type" : "array", + "items" : { + "type" : "number", + "format" : "float" + } + }, + "temporalQueryConfig" : { + "$ref" : "#/components/schemas/TemporalQueryConfig" + } + } + }, + "TemporalQueryConfig" : { + "type" : "object", + "properties" : { + "queryId" : { + "type" : "string" + }, + "hints" : { + "uniqueItems" : true, + "type" : "array", + "items" : { + "type" : "string", + "enum" : [ "exact", "inexact", "lsh", "ecp", "mi", "pq", "sh", "va", "vaf", "vav", "sequential", "empirical" ] + } }, "timeDistances" : { "type" : "array", @@ -1571,15 +1675,57 @@ "type" : "number", "format" : "float" }, - "metadataAccessSpec" : { + "computeTemporalObjects" : { + "type" : "boolean" + }, + "distance" : { + "type" : "string", + "enum" : [ "chisquared", "correlation", "cosine", "hamming", "jaccard", "kullbackleibler", "chebyshev", "euclidean", "squaredeuclidean", "manhattan", "minkowski", "spannorm", "haversine" ] + }, + "distanceWeights" : { "type" : "array", "items" : { - "$ref" : "#/components/schemas/MetadataAccessSpecification" + "type" : "number", + "format" : "float" } }, - "messageType" : { - "type" : "string", - "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + "norm" : { + "type" : "number", + "format" : "float" + }, + "resultsPerModule" : { + "type" : "integer", + "format" : "int32" + }, + "maxResults" : { + "type" : "integer", + "format" : "int32" + }, + "relevantSegmentIds" : { + "uniqueItems" : true, + "type" : "array", + "items" : { + "type" : "string" + } + }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "distanceWeightsIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "normIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "correspondenceFunction" : { + "$ref" : "#/components/schemas/CorrespondenceFunction" + }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "rawResultsPerModule" : { + "type" : "integer", + "format" : "int32" } } }, @@ -1705,6 +1851,9 @@ "Tag" : { "type" : "object", "properties" : { + "description" : { + "type" : "string" + }, "name" : { "type" : "string" }, @@ -1714,9 +1863,6 @@ }, "id" : { "type" : "string" - }, - "description" : { - "type" : "string" } } }, From 6cdd6ea81d6f5ab43f605da331ee688f78c69321 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Tue, 12 Apr 2022 17:15:12 +0200 Subject: [PATCH 57/72] bumping version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 44ee1019f..3df8e5c0e 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.10.2' + version = '3.11.1' apply plugin: 'java-library' apply plugin: 'maven-publish' From 80481f84b426af92ad82d8d05e119675d4c46c6a Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Tue, 12 Apr 2022 17:30:28 +0200 Subject: [PATCH 58/72] adressing comments from review --- .../api/messages/general/IntegerMessage.java | 18 +++++++++++ .../actions/bool/CountRowsGetHandler.java | 11 ++++--- .../FindSegmentFeaturesGetHandler.java | 3 +- .../FindObjectPaginationGetHandler.java | 6 ++-- .../vitrivr/cineast/core/db/DBSelector.java | 2 +- .../db/cottontaildb/CottontailSelector.java | 2 +- .../db/dao/reader/AbstractEntityReader.java | 2 +- docs/openapi.json | 32 ++++++++++++------- 8 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java new file mode 100644 index 000000000..8861c219d --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java @@ -0,0 +1,18 @@ +package org.vitrivr.cineast.api.messages.general; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IntegerMessage { + + private final int value; + + @JsonCreator + public IntegerMessage(@JsonProperty("value") int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java index 32a582abe..046911300 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java @@ -7,29 +7,30 @@ import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.api.messages.general.IntegerMessage; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; import org.vitrivr.cineast.standalone.config.Config; -public class CountRowsGetHandler implements GetRestHandler { +public class CountRowsGetHandler implements GetRestHandler { public static final String ROUTE = "count/table/{" + TABLE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(CountRowsGetHandler.class); @Override - public Integer doGet(Context ctx) { + public IntegerMessage doGet(Context ctx) { try (final var selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get()) { var tableName = ctx.pathParam(TABLE_NAME); selector.open(tableName); var count = selector.rowCount(); LOGGER.trace("counted {} objects in table {}", count, tableName); - return count; + return new IntegerMessage(count); } } @Override - public Class outClass() { - return Integer.class; + public Class outClass() { + return IntegerMessage.class; } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java index 5777e1159..554c7a50d 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java @@ -11,8 +11,7 @@ import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; import org.vitrivr.cineast.api.util.QueryUtil; -public class FindSegmentFeaturesGetHandler implements - GetRestHandler { +public class FindSegmentFeaturesGetHandler implements GetRestHandler { public static final String ROUTE = "find/feature/all/by/id/{" + ID_QUALIFIER + "}"; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java index f42f17ce3..4798408d0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -24,8 +24,10 @@ public class FindObjectPaginationGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); - final var skip = Integer.parseInt(parameters.get(SKIP_NAME)); - final var limit = Integer.parseInt(parameters.get(LIMIT_NAME)); + final var skipParam = parameters.get(SKIP_NAME); + final var skip = skipParam == null ? 0 : Integer.parseInt(skipParam); + final var limitParam = parameters.get(LIMIT_NAME); + final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(LIMIT_NAME); try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { var result = ol.getAllObjects(skip, limit); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index 6cf9c661e..553018b0b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -306,7 +306,7 @@ default List> getAll(String order, int skip, /** * SELECT count(*) FROM table */ - default Integer rowCount(){ + default int rowCount(){ return getAll().size(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 937836c03..4d992b932 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -370,7 +370,7 @@ public List> getAll(String order, int skip, i } @Override - public Integer rowCount() { + public int rowCount() { final Query query = new Query(this.fqn).count().queryId("count-star-"+this.fqn); return Math.toIntExact(this.cottontail.client.query(query).next().asLong(0)); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java index e6e2e794a..cf70686d8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java @@ -30,7 +30,7 @@ public void close() { this.selector.close(); } - public Integer rowCount(){ + public int rowCount(){ return this.selector.rowCount(); } } diff --git a/docs/openapi.json b/docs/openapi.json index 4d154d793..408ffdb86 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1090,8 +1090,7 @@ "content" : { "application/json" : { "schema" : { - "type" : "integer", - "format" : "int32" + "$ref" : "#/components/schemas/IntegerMessage" } } } @@ -1459,6 +1458,9 @@ "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceWeightsIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1468,9 +1470,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "distanceIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1630,14 +1629,14 @@ "$ref" : "#/components/schemas/MetadataAccessSpecification" } }, - "maxLength" : { - "type" : "number", - "format" : "float" - }, "messageType" : { "type" : "string", "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] }, + "maxLength" : { + "type" : "number", + "format" : "float" + }, "timeDistances" : { "type" : "array", "items" : { @@ -1711,6 +1710,9 @@ "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceWeightsIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1720,9 +1722,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "distanceIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -2033,6 +2032,15 @@ } } }, + "IntegerMessage" : { + "type" : "object", + "properties" : { + "value" : { + "type" : "integer", + "format" : "int32" + } + } + }, "Ping" : { "type" : "object", "properties" : { From 19ca74b1dbbaa44e29147757d770bacdf2c4a2c2 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Tue, 12 Apr 2022 17:40:22 +0200 Subject: [PATCH 59/72] not executing empty in-clauses to prevent db-exceptions --- .../cineast/core/db/cottontaildb/CottontailSelector.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index d7b668b7c..bb7c37c9b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -163,6 +163,10 @@ public List> getRows(String fieldName, Relati } private List> getRowsHelper(String fieldName, String op, Object[] mapped, String dbQueryID) { + if (op.equals("IN") && mapped.length == 0) { + LOGGER.debug("empty in-clause, not executing query {}", dbQueryID); + return new ArrayList<>(0); + } final Query query = new Query(this.fqn).select("*", null).where(new Expression(fieldName, op, mapped)).queryId(dbQueryID); try { return processResults(this.cottontail.client.query(query)); From 59d5b3494300732ba17158726e56d533aaddbf69 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Tue, 12 Apr 2022 17:41:43 +0200 Subject: [PATCH 60/72] bumping version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 44ee1019f..31d3fbfdf 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.10.2' + version = '3.11.0' apply plugin: 'java-library' apply plugin: 'maven-publish' From 0301b11bc3fb2c531a61dfef94030aa8cf9d70a8 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 10:29:59 +0200 Subject: [PATCH 61/72] re-generating openapi --- docs/openapi.json | 115 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 19 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index a817fa4a6..2f13e1dce 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1248,8 +1248,7 @@ "type" : "object", "properties" : { "objectid" : { - "type" : "string", - "writeOnly" : true + "type" : "string" }, "name" : { "type" : "string" @@ -1265,9 +1264,6 @@ "type" : "boolean", "writeOnly" : true }, - "objectId" : { - "type" : "string" - }, "contentURL" : { "type" : "string" } @@ -1351,8 +1347,7 @@ "type" : "object", "properties" : { "queryId" : { - "type" : "string", - "format" : "uuid" + "type" : "string" }, "hints" : { "uniqueItems" : true, @@ -1392,21 +1387,21 @@ "type" : "string" } }, - "correspondenceFunctionIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, - "correspondenceFunction" : { - "$ref" : "#/components/schemas/CorrespondenceFunction" - }, "distanceWeightsIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "normIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, + "correspondenceFunction" : { + "$ref" : "#/components/schemas/CorrespondenceFunction" + }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1418,7 +1413,7 @@ "properties" : { "type" : { "type" : "string", - "enum" : [ "IMAGE", "AUDIO", "MOTION", "MODEL3D", "LOCATION", "PARAMETERISED_LOCATION", "TIME", "TEXT", "TAG", "SEMANTIC", "ID", "BOOLEAN" ] + "enum" : [ "IMAGE", "AUDIO", "MODEL3D", "LOCATION", "PARAMETERISED_LOCATION", "TIME", "TEXT", "TAG", "SEMANTIC", "SKELETON", "ID", "BOOLEAN" ] }, "data" : { "type" : "string", @@ -1558,7 +1553,13 @@ } }, "config" : { - "$ref" : "#/components/schemas/QueryConfig" + "$ref" : "#/components/schemas/TemporalQueryConfig" + }, + "metadataAccessSpec" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/MetadataAccessSpecification" + } }, "timeDistances" : { "type" : "array", @@ -1567,19 +1568,95 @@ "format" : "float" } }, + "temporalQueryConfig" : { + "$ref" : "#/components/schemas/TemporalQueryConfig" + }, + "messageType" : { + "type" : "string", + "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + }, "maxLength" : { "type" : "number", "format" : "float" + } + } + }, + "TemporalQueryConfig" : { + "type" : "object", + "properties" : { + "queryId" : { + "type" : "string" }, - "metadataAccessSpec" : { + "hints" : { + "uniqueItems" : true, "type" : "array", "items" : { - "$ref" : "#/components/schemas/MetadataAccessSpecification" + "type" : "string", + "enum" : [ "exact", "inexact", "lsh", "ecp", "mi", "pq", "sh", "va", "vaf", "vav", "sequential", "empirical" ] } }, - "messageType" : { + "timeDistances" : { + "type" : "array", + "items" : { + "type" : "number", + "format" : "float" + } + }, + "maxLength" : { + "type" : "number", + "format" : "float" + }, + "computeTemporalObjects" : { + "type" : "boolean" + }, + "distance" : { "type" : "string", - "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + "enum" : [ "chisquared", "correlation", "cosine", "hamming", "jaccard", "kullbackleibler", "chebyshev", "euclidean", "squaredeuclidean", "manhattan", "minkowski", "spannorm", "haversine" ] + }, + "distanceWeights" : { + "type" : "array", + "items" : { + "type" : "number", + "format" : "float" + } + }, + "norm" : { + "type" : "number", + "format" : "float" + }, + "resultsPerModule" : { + "type" : "integer", + "format" : "int32" + }, + "maxResults" : { + "type" : "integer", + "format" : "int32" + }, + "relevantSegmentIds" : { + "uniqueItems" : true, + "type" : "array", + "items" : { + "type" : "string" + } + }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "distanceWeightsIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "normIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "correspondenceFunction" : { + "$ref" : "#/components/schemas/CorrespondenceFunction" + }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, + "rawResultsPerModule" : { + "type" : "integer", + "format" : "int32" } } }, From 8a73254732ac11995f66dbaa99085027fad61d92 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 10:46:24 +0200 Subject: [PATCH 62/72] re-generating openapi --- docs/openapi.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 408ffdb86..298bf8e8c 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1455,9 +1455,6 @@ "type" : "string" } }, - "correspondenceFunctionIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1470,6 +1467,9 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1707,9 +1707,6 @@ "type" : "string" } }, - "correspondenceFunctionIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1722,6 +1719,9 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" From ee2f54e0bba44de22ee609c5a0199585c5e0ff18 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 11:01:10 +0200 Subject: [PATCH 63/72] fixing bug from refactoring --- .../actions/mediaobject/FindObjectPaginationGetHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java index 4798408d0..69be9dcb8 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -27,11 +27,11 @@ public MediaObjectQueryResult doGet(Context ctx) { final var skipParam = parameters.get(SKIP_NAME); final var skip = skipParam == null ? 0 : Integer.parseInt(skipParam); final var limitParam = parameters.get(LIMIT_NAME); - final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(LIMIT_NAME); + final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(limitParam); try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { var result = ol.getAllObjects(skip, limit); - LOGGER.trace("returning {} elements for limit {} and skip {}", result.size(), limit, skip); + LOGGER.trace("returning {} elements for skip {} and limit {}", result.size(), skip, limit); return new MediaObjectQueryResult("", result); } } From a435c71497bff09b14daf70cf82e79d6765d4e9f Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 13 Apr 2022 11:28:25 +0200 Subject: [PATCH 64/72] Fixed index creation in light of recent changes to CreateIndex message. --- .../core/db/cottontaildb/CottontailEntityCreator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index b49aa0621..f9d1f44f4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -359,9 +359,9 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { private boolean createIndex(String entityName, String attribute, IndexType type, long txId) { - var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - final String indexName = fqn + ".idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); + final String entityNameFqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; + final String indexName = "idx_" + attribute + "_" + type.name().toLowerCase(); + final CreateIndex index = new CreateIndex(entityNameFqn, attribute, type).column(attribute).name(indexName).txId(txId); try { this.cottontail.client.create(index); } catch (StatusRuntimeException e) { From e65970eec4808bb8bb9387d95dea81ca6c5564ba Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 12:13:04 +0200 Subject: [PATCH 65/72] updating for index-structures --- .../MediaObjectMetadataDescriptor.java | 16 +++++++------- .../cottontaildb/CottontailEntityCreator.java | 5 ++--- .../dao/writer/MediaObjectMetadataWriter.java | 2 +- docs/openapi.json | 22 ++++++++----------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java index 2ea2b8c54..f9f9631cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java @@ -48,7 +48,7 @@ public class MediaObjectMetadataDescriptor implements ExistenceCheck { /** * ID of the MultimediaObject this MediaObjectMetadataDescriptor belongs to. */ - private final String objectId; + private final String objectid; /** * String value that identifies the metadata domain (e.g. EXIF, IPTC, DC...) @@ -95,7 +95,7 @@ public MediaObjectMetadataDescriptor( @JsonProperty(KEY_COL_NAME) String key, @JsonProperty(VAL_COL_NAME) @Nullable Object value, @JsonProperty(value = "exists", defaultValue = "false") boolean exists) { - this.objectId = objectId; + this.objectid = objectId; this.key = key; this.domain = domain; this.exists = exists; @@ -135,7 +135,7 @@ static boolean isSupportedValue(Object value) { */ public MediaObjectMetadataDescriptor(Map data) throws DatabaseLookupException { if (data.get(FIELDNAMES[0]) != null && data.get(FIELDNAMES[0]).getType() == ProviderDataType.STRING) { - this.objectId = data.get(FIELDNAMES[0]).getString(); + this.objectid = data.get(FIELDNAMES[0]).getString(); } else { throw new DatabaseLookupException("Could not read column '" + FIELDNAMES[0] + "' for MediaObjectDescriptor."); } @@ -156,9 +156,9 @@ public MediaObjectMetadataDescriptor(Map data) th this.exists = true; } - @JsonProperty - public String getObjectId() { - return objectId; + @JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) + public String getObjectid() { + return objectid; } @JsonProperty @@ -212,12 +212,12 @@ public boolean equals(Object o) { return false; } MediaObjectMetadataDescriptor that = (MediaObjectMetadataDescriptor) o; - return exists == that.exists && Objects.equals(objectId, that.objectId) && Objects.equals(domain, that.domain) && Objects.equals(key, that.key) && Objects.equals(value, that.value); + return exists == that.exists && Objects.equals(objectid, that.objectid) && Objects.equals(domain, that.domain) && Objects.equals(key, that.key) && Objects.equals(value, that.value); } @Override public int hashCode() { - return Objects.hash(objectId, domain, key, value, exists); + return Objects.hash(objectid, domain, key, value, exists); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index b49aa0621..051927434 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -360,13 +360,12 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { private boolean createIndex(String entityName, String attribute, IndexType type, long txId) { var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - final String indexName = fqn + ".idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); + final CreateIndex index = new CreateIndex(fqn, attribute, type).column(entityName + "." + attribute).txId(txId); try { this.cottontail.client.create(index); } catch (StatusRuntimeException e) { if (e.getStatus().getCode() == Status.ALREADY_EXISTS.getCode()) { - LOGGER.warn("Index {} was not created because it already exists", indexName); + LOGGER.warn("Index on entity {}, attribute {}, type {} was not created because it already exists", entityName, attribute, type); return false; } throw e; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java index 53a507336..ef2041adc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java @@ -29,7 +29,7 @@ protected PersistentTuple generateTuple(MediaObjectMetadataDescriptor entity) { if (entity.getValueProvider() instanceof NothingProvider) { return null; } - return this.writer.generateTuple(entity.getObjectId(), entity.getDomain(), entity.getKey(), + return this.writer.generateTuple(entity.getObjectid(), entity.getDomain(), entity.getKey(), entity.getValue()); } } diff --git a/docs/openapi.json b/docs/openapi.json index 298bf8e8c..0b8971d8c 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1162,8 +1162,7 @@ "type" : "object", "properties" : { "objectid" : { - "type" : "string", - "writeOnly" : true + "type" : "string" }, "domain" : { "type" : "string" @@ -1177,9 +1176,6 @@ "exists" : { "type" : "boolean", "writeOnly" : true - }, - "objectId" : { - "type" : "string" } } }, @@ -1246,15 +1242,15 @@ "type" : "string" } }, + "messageType" : { + "type" : "string", + "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + }, "idList" : { "type" : "array", "items" : { "type" : "string" } - }, - "messageType" : { - "type" : "string", - "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] } } }, @@ -1455,7 +1451,7 @@ "type" : "string" } }, - "distanceIfEmpty" : { + "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "distanceWeightsIfEmpty" : { @@ -1467,7 +1463,7 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "correspondenceFunctionIfEmpty" : { + "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "rawResultsPerModule" : { @@ -1707,7 +1703,7 @@ "type" : "string" } }, - "distanceIfEmpty" : { + "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "distanceWeightsIfEmpty" : { @@ -1719,7 +1715,7 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "correspondenceFunctionIfEmpty" : { + "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, "rawResultsPerModule" : { From fa70d91c657aa718c0bd85de4c21958b20fb1b15 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 15:29:55 +0200 Subject: [PATCH 66/72] enhanced logging, not setting name explicitly --- .../core/db/cottontaildb/CottontailEntityCreator.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index f9d1f44f4..fe64cc3ec 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -359,14 +359,13 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { private boolean createIndex(String entityName, String attribute, IndexType type, long txId) { - final String entityNameFqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - final String indexName = "idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(entityNameFqn, attribute, type).column(attribute).name(indexName).txId(txId); + var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; + final CreateIndex index = new CreateIndex(fqn, attribute, type).txId(txId); try { this.cottontail.client.create(index); } catch (StatusRuntimeException e) { if (e.getStatus().getCode() == Status.ALREADY_EXISTS.getCode()) { - LOGGER.warn("Index {} was not created because it already exists", indexName); + LOGGER.warn("Index on entity {}, attribute {}, type {} was not created because it already exists", entityName, attribute, type); return false; } throw e; From 9591b8a959c69dc7470c5cb859cfb7a099dd3cfc Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Wed, 13 Apr 2022 16:06:58 +0200 Subject: [PATCH 67/72] returning empty result on error which is consistent with rest api behavior --- .../FindObjectPaginationGetHandler.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java index 69be9dcb8..5177f3745 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -6,6 +6,7 @@ import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import java.util.ArrayList; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -24,15 +25,18 @@ public class FindObjectPaginationGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); - final var skipParam = parameters.get(SKIP_NAME); - final var skip = skipParam == null ? 0 : Integer.parseInt(skipParam); - final var limitParam = parameters.get(LIMIT_NAME); - final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(limitParam); - try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final var skipParam = parameters.get(SKIP_NAME); + final var skip = skipParam == null ? 0 : Integer.parseInt(skipParam); + final var limitParam = parameters.get(LIMIT_NAME); + final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(limitParam); + var result = ol.getAllObjects(skip, limit); LOGGER.trace("returning {} elements for skip {} and limit {}", result.size(), skip, limit); return new MediaObjectQueryResult("", result); + } catch (Exception e) { + LOGGER.error("Error during request", e); + return new MediaObjectQueryResult("", new ArrayList<>()); } } From 660967d014406d139b2f13674fc9f6aba7622052 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Fri, 22 Apr 2022 12:28:12 +0200 Subject: [PATCH 68/72] Adjusted version of Cottontail DB to prevent failure of integration tests. --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 24eed03d1..065e4fd42 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -17,7 +17,7 @@ jobs: # Setup Cottontail DB service container services: cottontail: - image: vitrivr/cottontaildb:0.13.3 + image: vitrivr/cottontaildb:0.14.1 ports: - 1865:1865 options: -it From d9df3aae3b3cf81e79e469f314b5d299378e2cbf Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 25 Apr 2022 08:05:14 +0200 Subject: [PATCH 69/72] bumping cottontail version for CI --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 065e4fd42..f36ba4013 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -17,7 +17,7 @@ jobs: # Setup Cottontail DB service container services: cottontail: - image: vitrivr/cottontaildb:0.14.1 + image: vitrivr/cottontaildb:0.15.0 ports: - 1865:1865 options: -it From 3b156d657426c36e6fbc60bdd57247b07b955437 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 25 Apr 2022 14:11:01 +0200 Subject: [PATCH 70/72] re-generating openapi --- docs/openapi.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 2f13e1dce..0c6cf3ac3 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1782,6 +1782,9 @@ "Tag" : { "type" : "object", "properties" : { + "description" : { + "type" : "string" + }, "name" : { "type" : "string" }, @@ -1791,9 +1794,6 @@ }, "id" : { "type" : "string" - }, - "description" : { - "type" : "string" } } }, From 3cf3b1bd7ae9b33051536d8dd883e189dc2a5abc Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 25 Apr 2022 14:12:57 +0200 Subject: [PATCH 71/72] re-generating openapi --- docs/openapi.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 0b8971d8c..6e09facea 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1242,15 +1242,15 @@ "type" : "string" } }, - "messageType" : { - "type" : "string", - "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] - }, "idList" : { "type" : "array", "items" : { "type" : "string" } + }, + "messageType" : { + "type" : "string", + "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] } } }, @@ -1454,6 +1454,9 @@ "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceWeightsIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1463,9 +1466,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "distanceIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1706,6 +1706,9 @@ "correspondenceFunctionIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, + "distanceIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceWeightsIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1715,9 +1718,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "distanceIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" From 6b4b41b1607eeaf9fb6ea3315d9d0c80b3a88c9e Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 25 Apr 2022 16:02:35 +0200 Subject: [PATCH 72/72] cleanup, improving example config, fixing bugs which caused default extraction to not correctly resolve --- .../FileSystemThumbnailResolver.java | 27 +++++++----- .../db/cottontaildb/CottontailWriter.java | 7 ++- ...SimpleFulltextFeatureDescriptorWriter.java | 3 ++ .../cineast/core/features/TagsFtSearch.java | 15 ------- .../abstracts/AbstractTextRetriever.java | 2 +- .../standalone/cli/TextRetrievalCommand.java | 2 - .../cineast/standalone/config/Config.java | 3 +- .../standalone/config/MetadataConfig.java | 3 +- .../monitoring/PrometheusServer.java | 2 +- cineast.json | 17 +++----- example_json_job.json | 43 ------------------- extraction_config.json | 34 ++++++++++++--- 12 files changed, 65 insertions(+), 93 deletions(-) delete mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java delete mode 100644 example_json_job.json diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java index 43c17075c..97cd27c0d 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java @@ -2,11 +2,14 @@ import java.io.File; import java.io.FileNotFoundException; +import org.apache.logging.log4j.LogManager; public class FileSystemThumbnailResolver implements ThumbnailResolver { private final File baseFolder; + private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); + public FileSystemThumbnailResolver(File baseFolder) { this.baseFolder = baseFolder; } @@ -15,23 +18,27 @@ public FileSystemThumbnailResolver(File baseFolder) { public ResolutionResult resolve(String segmentId) { if (segmentId == null) { + LOGGER.error("no segment id provided"); return null; } String[] split = segmentId.split("_"); if (split.length < 3) { + LOGGER.error("invalid segment id {}", segmentId); return null; } File[] candidates = new File[]{ - new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".jpg"), - new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".png"), - new File(baseFolder, split[1] + "/" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/" + split[2] + ".png"), - new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".png"), - new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".png"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + segmentId + ".jpg"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + segmentId + ".png"), + new File(baseFolder, split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".png"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".png"), }; for (File candidate : candidates) { @@ -39,12 +46,12 @@ public ResolutionResult resolve(String segmentId) { try { return new ResolutionResult(candidate); } catch (FileNotFoundException e) { - e.printStackTrace(); + LOGGER.error(e); return null; } } } - + LOGGER.error("no thumbnail found for segment id {}", segmentId); return null; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index e0d7f0643..efa92ecaa 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -2,6 +2,8 @@ import io.grpc.StatusRuntimeException; import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; @@ -59,6 +61,9 @@ public boolean exists(String key, String value) { @Override public boolean persist(List tuples) { + if (this.fqn == null) { + LOGGER.warn("fqn was null, not inserting {} tuples {}", tuples.size(), StringUtils.join(tuples, ", ")); + } long start = System.currentTimeMillis(); int size = tuples.size(); long txId = 0L; @@ -84,7 +89,7 @@ public boolean persist(List tuples) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); this.cottontail.client.insert(insert); insert = new BatchInsert().into(this.fqn).columns(this.names); - if(useTransactions){ + if (useTransactions) { insert.txId(txId); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java index 9e9127af3..42a4b1d4a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java @@ -10,6 +10,9 @@ public class SimpleFulltextFeatureDescriptorWriter extends AbstractBatchedEntity public SimpleFulltextFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { super(writer); + if (entityname == null) { + throw new IllegalArgumentException("An entity name cannot be null"); + } this.entityname = entityname; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java deleted file mode 100644 index 084153b94..000000000 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.vitrivr.cineast.core.features; - -import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; - -public class TagsFtSearch extends AbstractTextRetriever { - - public static final String TAGS_FT_TABLE_NAME = "features_tagsft"; - - /** - * Default constructor for {@link TagsFtSearch}. - */ - public TagsFtSearch() { - super(TagsFtSearch.TAGS_FT_TABLE_NAME); - } -} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index cb06aa87b..30d33f1b9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -26,7 +26,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; -import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.dao.writer.SimpleFulltextFeatureDescriptorWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -79,6 +78,7 @@ public void init(DBSelectorSupplier selectorSupply) { @Override public void init(PersistencyWriterSupplier phandlerSupply) { this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.tableName); + writer.init(); } @Override diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java index a9c954726..4004d6fe0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java @@ -6,10 +6,8 @@ import java.util.List; import org.vitrivr.cineast.core.data.query.containers.TextQueryTermContainer; import org.vitrivr.cineast.core.features.AudioTranscriptionSearch; -import org.vitrivr.cineast.core.features.DescriptionTextSearch; import org.vitrivr.cineast.core.features.OCRSearch; import org.vitrivr.cineast.core.features.SubtitleFulltextSearch; -import org.vitrivr.cineast.core.features.TagsFtSearch; import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.util.ContinuousRetrievalLogic; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java index 485801759..1afb60ddd 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java @@ -54,8 +54,9 @@ public static Config loadConfig(String name) { LOGGER.warn("Could not read config file '{}'.", name); return null; } else { - LOGGER.info("Config file loaded!"); + LOGGER.trace("Config file loaded!"); initSharedConfig(config); + LOGGER.trace("Config initialized!"); return config; } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java index 370e23592..a4f691fad 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java @@ -43,7 +43,6 @@ public void setProperties(HashMap properties) { @JsonIgnore public MetadataExtractor getMetadataExtractor() { - MetadataExtractor extractor = ReflectionHelper.newMetadataExtractor(this.name); - return extractor; + return ReflectionHelper.newMetadataExtractor(this.name); } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java index 7e2dad6d4..bd14827fe 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java @@ -32,7 +32,7 @@ public static synchronized void initialize() { return; } if (!Config.sharedConfig().getMonitoring().enablePrometheus) { - LOGGER.info("Prometheus monitoring not enabled"); + LOGGER.trace("Prometheus monitoring not enabled"); lock.release(); return; } diff --git a/cineast.json b/cineast.json index 577af8e98..ccbcab07b 100644 --- a/cineast.json +++ b/cineast.json @@ -15,25 +15,18 @@ "globalcolor": [ {"feature": "AverageColor", "weight": 2.3}, {"feature": "MedianColor", "weight": 1.2}, - {"feature": "AverageFuzzyHist", "weight": 0.7}, - {"feature": "HueHistogram", "weight": 0.7}, - {"feature": "MedianFuzzyHist", "weight": 1.3 } + {"feature": "AverageFuzzyHist", "weight": 0.7} ], "localcolor": [ {"feature": "AverageColorARP44", "weight": 0.5}, - {"feature": "AverageColorARP44Normalized", "weight": 0.5}, - {"feature": "SubDivMedianFuzzyColor", "weight": 0.85}, {"feature": "AverageColorGrid8", "weight": 1.8}, - {"feature": "AverageColorGrid8Normalized", "weight": 1.8}, {"feature": "CLD", "weight": 1.3}, - {"feature": "CLDNormalized", "weight": 1.3}, {"feature": "MedianColorGrid8", "weight": 1.7}, {"feature": "AverageColorRaster", "weight": 1.0} ], "edge": [ {"feature": "EHD", "weight": 0.7}, - {"feature": "DominantEdgeGrid16", "weight": 1.4}, - {"feature": "DominantEdgeGrid8", "weight": 1.4} + {"feature": "DominantEdgeGrid16", "weight": 1.4} ], "localfeatures": [ {"feature": "HOGMirflickr25K512", "weight": 1.0} @@ -106,9 +99,9 @@ "serveContent": true, "serveUI": false, "uiLocation": "../vitrivr-ng/dist", - "thumbnailLocation": "/Volumes/V3C1/V3C1/thumbnails", - "objectLocation": "/Volumes/V3C1/V3C1/videos", - "objectsFilesAreIDed": true + "thumbnailLocation": "./thumbnails", + "objectLocation": "./data/", + "objectsFilesAreIDed": false }, "monitoring":{ diff --git a/example_json_job.json b/example_json_job.json deleted file mode 100644 index 10f0a1b49..000000000 --- a/example_json_job.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "input":{ - "path": "data/", - "depth": 3, - "skip": 0, - "id": { - "name": "SequentialObjectIdGenerator", - "properties": {} - } - }, - "extractors":[ - {"name": "AverageColor"}, - {"name": "AverageColorARP44"}, - {"name": "AverageColorCLD"}, - {"name": "AverageColorGrid8"}, - {"name": "AverageColorRaster"}, - {"name": "AverageFuzzyHist"}, - {"name": "AverageColorGrid8Reduced15"}, - {"name": "CLD"}, - {"name": "EdgeARP88"}, - {"name": "EdgeGrid16"}, - {"name": "EHD"}, - {"name": "HPCP12Shingle"}, - {"name": "MedianColor"}, - {"name": "MedianColorGrid8"}, - {"name": "HOGMirflickr25K512"}, - {"name": "SURFMirflickr25K512"}, - {"name": "VisualTextCoEmbedding"} - ], - "exporters":[ - { - "name": "ShotThumbnailsExporter", - "properties": { - "destination":"thumbnails/" - } - } - ], - "database":{ - "host": "output/", - "writer": "JSON", - "selector": "NONE" - } -} diff --git a/extraction_config.json b/extraction_config.json index ce6666813..008d50d7f 100644 --- a/extraction_config.json +++ b/extraction_config.json @@ -1,17 +1,41 @@ { - "input": { - "path": "videos", - "depth": 1, + "input":{ + "path": "data/", + "depth": 3, + "skip": 0, "id": { "name": "SequentialObjectIdGenerator", "properties": {} } }, - "extractors": [ + "extractors":[ {"name": "AverageColor"}, + {"name": "AverageColorARP44"}, + {"name": "AverageColorGrid8"}, + {"name": "AverageColorRaster"}, + {"name": "AverageFuzzyHist"}, + {"name": "CLD"}, + {"name": "DominantEdgeGrid16"}, + {"name": "EHD"}, + {"name": "HPCP12Shingle"}, {"name": "MedianColor"}, + {"name": "MedianColorGrid8"}, {"name": "OCRSearch"}, - {"name": "SubtitleFulltextSearch"} + {"name": "HOGMirflickr25K512"}, + {"name": "SURFMirflickr25K512"}, + {"name": "VisualTextCoEmbedding"} + ], + "metadata": [ + {"name": "TechnicalVideoMetadataExtractor"}, + {"name": "EXIFMetadataExtractor"} + ], + "exporters":[ + { + "name": "ShotThumbnailsExporter", + "properties": { + "destination":"thumbnails/" + } + } ], "database": { "writer": "COTTONTAIL",