diff --git a/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/clip/CLIP.kt b/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/clip/CLIP.kt index 58bf8ab9..476f281a 100644 --- a/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/clip/CLIP.kt +++ b/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/clip/CLIP.kt @@ -1,7 +1,5 @@ package org.vitrivr.engine.base.features.external.implementations.clip -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import org.vitrivr.engine.base.features.external.common.ExternalWithFloatVectorDescriptorAnalyser import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext @@ -9,7 +7,6 @@ import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.operators.Operator @@ -24,59 +21,26 @@ import java.util.* * @version 1.0.0 */ class CLIP : ExternalWithFloatVectorDescriptorAnalyser>() { - - companion object { - private val logger: KLogger = KotlinLogging.logger {} - } - override val contentClasses = setOf(ImageContent::class, TextContent::class) override val descriptorClass = FloatVectorDescriptor::class - // Default values for external API -// override val endpoint: String = "/extract/clip_image" -// override val host: String = "localhost" -// override val port: Int = 8888 - - // Size and list for prototypical descriptor - override val size = 512 - override val featureList = List(size) { 0.0f } - - /** - * Requests the CLIP feature descriptor for the given [ContentElement]. - * - * @param content The [ContentElement] for which to request the CLIP feature descriptor. - * @return A list of CLIP feature descriptors. - */ - override fun requestDescriptor(content: ContentElement<*>): List { - - //TODO make endpoints configurable - return when(content) { - is ImageContent -> httpRequest(content, "http://localhost:8888/extract/clip_image") - is TextContent -> httpRequest(content, "http://localhost:8888/extract/clip_text") - else -> throw IllegalArgumentException("Content '$content' not supported") - } - - - } - /** * Generates a prototypical [FloatVectorDescriptor] for this [CLIP]. * - * @param field [Schema.Field] to create the prototype for. * @return [FloatVectorDescriptor] */ - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), this.featureList, true) + override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { 0.0f }, true) /** - * Generates and returns a new [Extractor] instance for this [Analyser]. + * Generates and returns a new [Extractor] instance for this [CLIP]. * * @param field The [Schema.Field] to create an [Extractor] for. * @param input The [Operator] that acts as input to the new [Extractor]. * @param context The [IndexContext] to use with the [Extractor]. * @param persisting True, if the results of the [Extractor] should be persisted. * - * @return A new [Extractor] instance for this [Analyser] - * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + * @return A new [Extractor] instance for this [CLIP] + * @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Extractor] instance. */ override fun newExtractor( field: Schema.Field, FloatVectorDescriptor>, @@ -85,27 +49,23 @@ class CLIP : ExternalWithFloatVectorDescriptorAnalyser>() { persisting: Boolean, parameters: Map ): Extractor, FloatVectorDescriptor> { - require(field.analyser == this) { "" } - logger.debug { "Creating new CLIPImageExtractor for field '${field.fieldName}' with parameters $parameters." } - return CLIPExtractor(input, field, persisting, this) + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + return CLIPExtractor(input, field, persisting) } /** * Generates and returns a new [Retriever] instance for this [CLIP]. * * @param field The [Schema.Field] to create an [Retriever] for. - * @param content An array of [ContentElement]s to use with the [Retriever] + * @param content An array of [ContentElement] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] * - * @return A new [Retriever] instance for this [Analyser] - * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. + * @return A new [Retriever] instance for this [CLIP] + * @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance. */ - override fun newRetrieverForContent( - field: Schema.Field, FloatVectorDescriptor>, - content: Collection>, - context: QueryContext - ): Retriever, FloatVectorDescriptor> { - return this.newRetrieverForDescriptors(field, this.processContent(content), context) + override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { + val host = field.parameters[HOST_PARAMETER_NAME] ?: HOST_PARAMETER_DEFAULT + return this.newRetrieverForDescriptors(field, content.map { this.analyse(it, host) }, context) } /** @@ -115,15 +75,24 @@ class CLIP : ExternalWithFloatVectorDescriptorAnalyser>() { * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] * - * @return A new [Retriever] instance for this [Analyser] - * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. + * @return A new [Retriever] instance for this [CLIP] + * @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance. */ - override fun newRetrieverForDescriptors( - field: Schema.Field, FloatVectorDescriptor>, - descriptors: Collection, - context: QueryContext - ): Retriever, FloatVectorDescriptor> { - require(field.analyser == this) { } + override fun newRetrieverForDescriptors(field: Schema.Field, FloatVectorDescriptor>, descriptors: Collection, context: QueryContext): Retriever, FloatVectorDescriptor> { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } return CLIPRetriever(field, descriptors.first(), context) } + + /** + * Requests the CLIP feature descriptor for the given [ContentElement]. + * + * @param content The [ContentElement] for which to request the CLIP feature descriptor. + * @param hostname The hostname of the external feature descriptor service. + * @return A list of CLIP feature descriptors. + */ + override fun analyse(content: ContentElement<*>, hostname: String): FloatVectorDescriptor = when (content) { + is ImageContent -> FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/clip_image"), true) + is TextContent -> FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/clip_text"), true) + else -> throw IllegalArgumentException("Content '$content' not supported") + } } \ No newline at end of file diff --git a/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dino/DINO.kt b/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dino/DINO.kt index d941f788..65b9c945 100644 --- a/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dino/DINO.kt +++ b/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dino/DINO.kt @@ -9,6 +9,7 @@ import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser @@ -32,43 +33,27 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser() { override val contentClasses = setOf(ImageContent::class) override val descriptorClass = FloatVectorDescriptor::class - // Default values for external API - private val endpoint: String = "http://localhost:8888/extract/dino" //TODO make configurable - - // Size and list for prototypical descriptor - override val size = 384 - override val featureList = List(size) { 0.0f } - - /** - * Requests the DINO feature descriptor for the given [ContentElement]. + * Generates a prototypical [FloatVectorDescriptor] for this [DINO]. * - * @param content The [ContentElement] for which to request the DINO feature descriptor. - * @return A list of DINO feature descriptors. + * @return [FloatVectorDescriptor] */ - override fun requestDescriptor(content: ContentElement<*>): List { - return httpRequest(content, endpoint) - } + override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(384) { 0.0f }, true) /** - * Generates a prototypical [FloatVectorDescriptor] for this [DINO]. + * Generates and returns a new [Extractor] instance for this [DINO]. * - * @param field [Schema.Field] to create the prototype for. - * @return [FloatVectorDescriptor] + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * @param persisting True, if the results of the [Extractor] should be persisted. + * + * @return A new [Extractor] instance for this [DINO] + * @throws [UnsupportedOperationException], if this [DINO] does not support the creation of an [Extractor] instance. */ - override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), featureList, true) - - - override fun newExtractor( - field: Schema.Field, - input: Operator, - context: IndexContext, - persisting: Boolean, - parameters: Map - ): Extractor { - require(field.analyser == this) { "" } - logger.debug { "Creating new DINOExtractor for field '${field.fieldName}' with parameters $parameters." } - return DINOExtractor(input, field, persisting, this) + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext, persisting: Boolean, parameters: Map): Extractor { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + return DINOExtractor(input, field, persisting) } /** @@ -81,13 +66,10 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser() { * @return A new [Retriever] instance for this [Analyser] * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. */ - override fun newRetrieverForContent( - field: Schema.Field, - content: Collection, - context: QueryContext - ): Retriever { + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): Retriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - return this.newRetrieverForDescriptors(field, this.processContent(content), context) + val host = field.parameters[HOST_PARAMETER_NAME] ?: HOST_PARAMETER_DEFAULT + return this.newRetrieverForDescriptors(field, content.map { this.analyse(it, host) }, context) } /** @@ -100,12 +82,19 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser() { * @return A new [Retriever] instance for this [Analyser] * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. */ - override fun newRetrieverForDescriptors( - field: Schema.Field, - descriptors: Collection, - context: QueryContext - ): Retriever { + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): Retriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } return DINORetriever(field, descriptors.first(), context) } + + /** + * Requests the CLIP feature descriptor for the given [ContentElement]. + * + * @param content The [ImageContent] for which to request the [DINO] feature descriptor. + * @param hostname The hostname of the external feature descriptor service. + * @return A list of CLIP feature descriptors. + */ + override fun analyse(content: ImageContent, hostname: String): FloatVectorDescriptor { + return FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/dino"), true) + } } \ No newline at end of file