Skip to content

Commit

Permalink
Fixes merge issue with DINO and CLIP feature.
Browse files Browse the repository at this point in the history
  • Loading branch information
Ralph Gasser committed Feb 2, 2024
1 parent f34e7c0 commit fb361f5
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 101 deletions.
89 changes: 29 additions & 60 deletions ...es/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/clip/CLIP.kt
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package org.vitrivr.engine.base.features.external.implementations.clip

import io.github.oshai.kotlinlogging.KLogger
import io.github.oshai.kotlinlogging.KotlinLogging
import org.vitrivr.engine.base.features.external.common.ExternalWithFloatVectorDescriptorAnalyser
import org.vitrivr.engine.core.context.IndexContext
import org.vitrivr.engine.core.context.QueryContext
import org.vitrivr.engine.core.model.content.element.ContentElement
import org.vitrivr.engine.core.model.content.element.ImageContent
import org.vitrivr.engine.core.model.content.element.TextContent
import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor
import org.vitrivr.engine.core.model.metamodel.Analyser
import org.vitrivr.engine.core.model.metamodel.Schema
import org.vitrivr.engine.core.model.retrievable.Retrievable
import org.vitrivr.engine.core.operators.Operator
Expand All @@ -24,59 +21,26 @@ import java.util.*
* @version 1.0.0
*/
class CLIP : ExternalWithFloatVectorDescriptorAnalyser<ContentElement<*>>() {

companion object {
private val logger: KLogger = KotlinLogging.logger {}
}

override val contentClasses = setOf(ImageContent::class, TextContent::class)
override val descriptorClass = FloatVectorDescriptor::class

// Default values for external API
// override val endpoint: String = "/extract/clip_image"
// override val host: String = "localhost"
// override val port: Int = 8888

// Size and list for prototypical descriptor
override val size = 512
override val featureList = List(size) { 0.0f }

/**
* Requests the CLIP feature descriptor for the given [ContentElement].
*
* @param content The [ContentElement] for which to request the CLIP feature descriptor.
* @return A list of CLIP feature descriptors.
*/
override fun requestDescriptor(content: ContentElement<*>): List<Float> {

//TODO make endpoints configurable
return when(content) {
is ImageContent -> httpRequest(content, "http://localhost:8888/extract/clip_image")
is TextContent -> httpRequest(content, "http://localhost:8888/extract/clip_text")
else -> throw IllegalArgumentException("Content '$content' not supported")
}


}

/**
* Generates a prototypical [FloatVectorDescriptor] for this [CLIP].
*
* @param field [Schema.Field] to create the prototype for.
* @return [FloatVectorDescriptor]
*/
override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), this.featureList, true)
override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { 0.0f }, true)

/**
* Generates and returns a new [Extractor] instance for this [Analyser].
* Generates and returns a new [Extractor] instance for this [CLIP].
*
* @param field The [Schema.Field] to create an [Extractor] for.
* @param input The [Operator] that acts as input to the new [Extractor].
* @param context The [IndexContext] to use with the [Extractor].
* @param persisting True, if the results of the [Extractor] should be persisted.
*
* @return A new [Extractor] instance for this [Analyser]
* @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance.
* @return A new [Extractor] instance for this [CLIP]
* @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Extractor] instance.
*/
override fun newExtractor(
field: Schema.Field<ContentElement<*>, FloatVectorDescriptor>,
Expand All @@ -85,27 +49,23 @@ class CLIP : ExternalWithFloatVectorDescriptorAnalyser<ContentElement<*>>() {
persisting: Boolean,
parameters: Map<String, Any>
): Extractor<ContentElement<*>, FloatVectorDescriptor> {
require(field.analyser == this) { "" }
logger.debug { "Creating new CLIPImageExtractor for field '${field.fieldName}' with parameters $parameters." }
return CLIPExtractor(input, field, persisting, this)
require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" }
return CLIPExtractor(input, field, persisting)
}

/**
* Generates and returns a new [Retriever] instance for this [CLIP].
*
* @param field The [Schema.Field] to create an [Retriever] for.
* @param content An array of [ContentElement]s to use with the [Retriever]
* @param content An array of [ContentElement] elements to use with the [Retriever]
* @param context The [QueryContext] to use with the [Retriever]
*
* @return A new [Retriever] instance for this [Analyser]
* @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance.
* @return A new [Retriever] instance for this [CLIP]
* @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance.
*/
override fun newRetrieverForContent(
field: Schema.Field<ContentElement<*>, FloatVectorDescriptor>,
content: Collection<ContentElement<*>>,
context: QueryContext
): Retriever<ContentElement<*>, FloatVectorDescriptor> {
return this.newRetrieverForDescriptors(field, this.processContent(content), context)
override fun newRetrieverForContent(field: Schema.Field<ContentElement<*>, FloatVectorDescriptor>, content: Collection<ContentElement<*>>, context: QueryContext): Retriever<ContentElement<*>, FloatVectorDescriptor> {
val host = field.parameters[HOST_PARAMETER_NAME] ?: HOST_PARAMETER_DEFAULT
return this.newRetrieverForDescriptors(field, content.map { this.analyse(it, host) }, context)
}

/**
Expand All @@ -115,15 +75,24 @@ class CLIP : ExternalWithFloatVectorDescriptorAnalyser<ContentElement<*>>() {
* @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever]
* @param context The [QueryContext] to use with the [Retriever]
*
* @return A new [Retriever] instance for this [Analyser]
* @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance.
* @return A new [Retriever] instance for this [CLIP]
* @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance.
*/
override fun newRetrieverForDescriptors(
field: Schema.Field<ContentElement<*>, FloatVectorDescriptor>,
descriptors: Collection<FloatVectorDescriptor>,
context: QueryContext
): Retriever<ContentElement<*>, FloatVectorDescriptor> {
require(field.analyser == this) { }
override fun newRetrieverForDescriptors(field: Schema.Field<ContentElement<*>, FloatVectorDescriptor>, descriptors: Collection<FloatVectorDescriptor>, context: QueryContext): Retriever<ContentElement<*>, FloatVectorDescriptor> {
require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" }
return CLIPRetriever(field, descriptors.first(), context)
}

/**
* Requests the CLIP feature descriptor for the given [ContentElement].
*
* @param content The [ContentElement] for which to request the CLIP feature descriptor.
* @param hostname The hostname of the external feature descriptor service.
* @return A list of CLIP feature descriptors.
*/
override fun analyse(content: ContentElement<*>, hostname: String): FloatVectorDescriptor = when (content) {
is ImageContent -> FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/clip_image"), true)
is TextContent -> FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/clip_text"), true)
else -> throw IllegalArgumentException("Content '$content' not supported")
}
}
71 changes: 30 additions & 41 deletions ...es/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dino/DINO.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import org.vitrivr.engine.core.context.QueryContext
import org.vitrivr.engine.core.model.content.Content
import org.vitrivr.engine.core.model.content.element.ContentElement
import org.vitrivr.engine.core.model.content.element.ImageContent
import org.vitrivr.engine.core.model.content.element.TextContent
import org.vitrivr.engine.core.model.descriptor.Descriptor
import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor
import org.vitrivr.engine.core.model.metamodel.Analyser
Expand All @@ -32,43 +33,27 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser<ImageContent>() {
override val contentClasses = setOf(ImageContent::class)
override val descriptorClass = FloatVectorDescriptor::class

// Default values for external API
private val endpoint: String = "http://localhost:8888/extract/dino" //TODO make configurable

// Size and list for prototypical descriptor
override val size = 384
override val featureList = List(size) { 0.0f }


/**
* Requests the DINO feature descriptor for the given [ContentElement].
* Generates a prototypical [FloatVectorDescriptor] for this [DINO].
*
* @param content The [ContentElement] for which to request the DINO feature descriptor.
* @return A list of DINO feature descriptors.
* @return [FloatVectorDescriptor]
*/
override fun requestDescriptor(content: ContentElement<*>): List<Float> {
return httpRequest(content, endpoint)
}
override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(384) { 0.0f }, true)

/**
* Generates a prototypical [FloatVectorDescriptor] for this [DINO].
* Generates and returns a new [Extractor] instance for this [DINO].
*
* @param field [Schema.Field] to create the prototype for.
* @return [FloatVectorDescriptor]
* @param field The [Schema.Field] to create an [Extractor] for.
* @param input The [Operator] that acts as input to the new [Extractor].
* @param context The [IndexContext] to use with the [Extractor].
* @param persisting True, if the results of the [Extractor] should be persisted.
*
* @return A new [Extractor] instance for this [DINO]
* @throws [UnsupportedOperationException], if this [DINO] does not support the creation of an [Extractor] instance.
*/
override fun prototype(field: Schema.Field<*,*>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), featureList, true)


override fun newExtractor(
field: Schema.Field<ImageContent, FloatVectorDescriptor>,
input: Operator<Retrievable>,
context: IndexContext,
persisting: Boolean,
parameters: Map<String, Any>
): Extractor<ImageContent, FloatVectorDescriptor> {
require(field.analyser == this) { "" }
logger.debug { "Creating new DINOExtractor for field '${field.fieldName}' with parameters $parameters." }
return DINOExtractor(input, field, persisting, this)
override fun newExtractor(field: Schema.Field<ImageContent, FloatVectorDescriptor>, input: Operator<Retrievable>, context: IndexContext, persisting: Boolean, parameters: Map<String, Any>): Extractor<ImageContent, FloatVectorDescriptor> {
require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" }
return DINOExtractor(input, field, persisting)
}

/**
Expand All @@ -81,13 +66,10 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser<ImageContent>() {
* @return A new [Retriever] instance for this [Analyser]
* @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance.
*/
override fun newRetrieverForContent(
field: Schema.Field<ImageContent, FloatVectorDescriptor>,
content: Collection<ImageContent>,
context: QueryContext
): Retriever<ImageContent, FloatVectorDescriptor> {
override fun newRetrieverForContent(field: Schema.Field<ImageContent, FloatVectorDescriptor>, content: Collection<ImageContent>, context: QueryContext): Retriever<ImageContent, FloatVectorDescriptor> {
require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" }
return this.newRetrieverForDescriptors(field, this.processContent(content), context)
val host = field.parameters[HOST_PARAMETER_NAME] ?: HOST_PARAMETER_DEFAULT
return this.newRetrieverForDescriptors(field, content.map { this.analyse(it, host) }, context)
}

/**
Expand All @@ -100,12 +82,19 @@ class DINO : ExternalWithFloatVectorDescriptorAnalyser<ImageContent>() {
* @return A new [Retriever] instance for this [Analyser]
* @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance.
*/
override fun newRetrieverForDescriptors(
field: Schema.Field<ImageContent, FloatVectorDescriptor>,
descriptors: Collection<FloatVectorDescriptor>,
context: QueryContext
): Retriever<ImageContent, FloatVectorDescriptor> {
override fun newRetrieverForDescriptors(field: Schema.Field<ImageContent, FloatVectorDescriptor>, descriptors: Collection<FloatVectorDescriptor>, context: QueryContext): Retriever<ImageContent, FloatVectorDescriptor> {
require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" }
return DINORetriever(field, descriptors.first(), context)
}

/**
* Requests the CLIP feature descriptor for the given [ContentElement].
*
* @param content The [ImageContent] for which to request the [DINO] feature descriptor.
* @param hostname The hostname of the external feature descriptor service.
* @return A list of CLIP feature descriptors.
*/
override fun analyse(content: ImageContent, hostname: String): FloatVectorDescriptor {
return FloatVectorDescriptor(UUID.randomUUID(), null, httpRequest(content, "$hostname/extract/dino"), true)
}
}

0 comments on commit fb361f5

Please sign in to comment.