[DERCBOT-1267] Gen AI - Document Compressor

bot/admin/server/src/main/kotlin/BotAdminService.kt

@@ -29,6 +29,7 @@ import ai.tock.bot.admin.bot.BotApplicationConfiguration
 import ai.tock.bot.admin.bot.BotApplicationConfigurationDAO
 import ai.tock.bot.admin.bot.BotConfiguration
 import ai.tock.bot.admin.bot.BotVersion
+import ai.tock.bot.admin.bot.compressor.BotDocumentCompressorConfigurationDAO
 import ai.tock.bot.admin.bot.observability.BotObservabilityConfigurationDAO
 import ai.tock.bot.admin.bot.rag.BotRAGConfiguration
 import ai.tock.bot.admin.bot.rag.BotRAGConfigurationDAO
@@ -83,6 +84,7 @@ object BotAdminService {
     private val ragConfigurationDAO: BotRAGConfigurationDAO get() = injector.provide()
     private val sentenceGenerationConfigurationDAO: BotSentenceGenerationConfigurationDAO get() = injector.provide()
     private val observabilityConfigurationDAO: BotObservabilityConfigurationDAO get() = injector.provide()
+    private val documentProcessorConfigurationDAO: BotDocumentCompressorConfigurationDAO get() = injector.provide()
     private val vectorStoreConfigurationDAO: BotVectorStoreConfigurationDAO get() = injector.provide()
     private val storyDefinitionDAO: StoryDefinitionConfigurationDAO get() = injector.provide()
     private val featureDAO: FeatureDAO get() = injector.provide()
@@ -1172,6 +1174,11 @@ object BotAdminService {
             }
         }
 
+        // delete the Document Compressor configuration
+        documentProcessorConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let {
+            documentProcessorConfigurationDAO.delete(it._id)
+        }
+
         // delete the Vector Store configuration
         vectorStoreConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let { config ->
             vectorStoreConfigurationDAO.delete(config._id)

bot/admin/server/src/main/kotlin/BotAdminVerticle.kt

@@ -57,7 +57,6 @@ import ai.tock.translator.I18nLabel
 import ai.tock.translator.Translator
 import ai.tock.translator.Translator.initTranslator
 import ai.tock.translator.TranslatorEngine
-import ch.tutteli.kbox.isNotNullAndNotBlank
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.github.salomonbrys.kodein.instance
 import io.vertx.core.http.HttpMethod.GET
@@ -491,6 +490,25 @@ open class BotAdminVerticle : AdminVerticle() {
             ObservabilityService.deleteConfig(context.organization, context.path("botId"))
         }
 
+        blockingJsonPost("/configuration/bots/:botId/document-compressor", admin) { context, configuration: BotDocumentCompressorConfigurationDTO  ->
+            if (context.organization == configuration.namespace) {
+                BotDocumentCompressorConfigurationDTO(DocumentCompressorService.saveDocumentCompressor(configuration))
+            } else {
+                unauthorized()
+            }
+        }
+
+        blockingJsonGet("/configuration/bots/:botId/document-compressor", admin) { context  ->
+            DocumentCompressorService.getDocumentCompressorConfiguration(context.organization, context.path("botId"))
+                ?.let {
+                    BotDocumentCompressorConfigurationDTO(it)
+                }
+        }
+
+        blockingDelete("/configuration/bots/:botId/document-compressor", admin) { context  ->
+            DocumentCompressorService.deleteConfig(context.organization, context.path("botId"))
+        }
+
         blockingJsonPost("/configuration/bots/:botId/vector-store", admin) { context, configuration: BotVectorStoreConfigurationDTO  ->
             if (context.organization == configuration.namespace) {
                 BotVectorStoreConfigurationDTO(VectorStoreService.saveVectorStore(configuration))

bot/admin/server/src/main/kotlin/model/BotDocumentCompressorConfigurationDTO.kt

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2017/2021 e-voyageurs technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.model
+
+
+import ai.tock.bot.admin.bot.compressor.BotDocumentCompressorConfiguration
+import ai.tock.genai.orchestratorcore.models.compressor.DocumentCompressorSetting
+import org.litote.kmongo.newId
+import org.litote.kmongo.toId
+
+data class BotDocumentCompressorConfigurationDTO(
+    val id: String? = null,
+    val namespace: String,
+    val botId: String,
+    val enabled: Boolean = false,
+    val setting: DocumentCompressorSetting,
+) {
+    constructor(configuration: BotDocumentCompressorConfiguration) : this(
+        id = configuration._id.toString(),
+        namespace = configuration.namespace,
+        botId = configuration.botId,
+        enabled = configuration.enabled,
+        setting = configuration.setting,
+    )
+
+    fun toBotDocumentCompressorConfiguration(): BotDocumentCompressorConfiguration =
+        BotDocumentCompressorConfiguration(
+            _id = id?.toId() ?: newId(),
+            namespace = namespace,
+            botId = botId,
+            enabled = enabled,
+            setting = setting,
+        )
+}
+
+
+

bot/admin/server/src/main/kotlin/service/DocumentCompressorService.kt

@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2017/2021 e-voyageurs technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.service
+
+import ai.tock.bot.admin.BotAdminService
+import ai.tock.bot.admin.bot.compressor.BotDocumentCompressorConfiguration
+import ai.tock.bot.admin.bot.compressor.BotDocumentCompressorConfigurationDAO
+import ai.tock.bot.admin.model.BotDocumentCompressorConfigurationDTO
+import ai.tock.shared.exception.rest.BadRequestException
+import ai.tock.shared.injector
+import ai.tock.shared.provide
+import ai.tock.shared.vertx.WebVerticle
+import com.mongodb.MongoWriteException
+import mu.KLogger
+import mu.KotlinLogging
+
+/**
+ * Service that manage the document compressor functionality
+ */
+object DocumentCompressorService {
+
+    private val logger: KLogger = KotlinLogging.logger {}
+    private val documentCompressorConfigurationDAO: BotDocumentCompressorConfigurationDAO get() = injector.provide()
+    /**
+     * Get the Document Compressor Configuration
+     * @param namespace: the namespace
+     * @param botId: the bot ID
+     */
+    fun getDocumentCompressorConfiguration(namespace: String, botId: String): BotDocumentCompressorConfiguration? {
+        return documentCompressorConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
+    }
+
+    /**
+     * Get the Document Compressor Configuration
+     * @param namespace: the namespace
+     * @param botId: the botId
+     * @param enabled: the document compressor activation (enabled or not)
+     */
+    fun getDocumentCompressorConfiguration(namespace: String, botId: String, enabled: Boolean): BotDocumentCompressorConfiguration? {
+        return documentCompressorConfigurationDAO.findByNamespaceAndBotIdAndEnabled(namespace, botId, enabled)
+    }
+
+    /**
+     * Deleting the Document Compressor Configuration
+     * @param namespace: the namespace
+     * @param botId: the bot ID
+     */
+    fun deleteConfig(namespace: String, botId: String) {
+        val documentCompressorConfig = documentCompressorConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
+            ?: WebVerticle.badRequest("No Document Compressor Configuration is defined yet [namespace: $namespace, botId: $botId]")
+        logger.info { "Deleting the Document Compressor Configuration [namespace: $namespace, botId: $botId]" }
+        return documentCompressorConfigurationDAO.delete(documentCompressorConfig._id)
+    }
+
+    /**
+     * Save Document Compressor Configuration and filter errors
+     * @param documentCompressorConfig : the document compressor configuration to create or update
+     * @throws [BadRequestException] if the document compressor configuration is invalid
+     * @return [BotDocumentCompressorConfiguration]
+     */
+    fun saveDocumentCompressor(
+        documentCompressorConfig: BotDocumentCompressorConfigurationDTO
+    ): BotDocumentCompressorConfiguration {
+        BotAdminService.getBotConfigurationsByNamespaceAndBotId(documentCompressorConfig.namespace, documentCompressorConfig.botId).firstOrNull()
+            ?: WebVerticle.badRequest("No bot configuration is defined yet [namespace: ${documentCompressorConfig.namespace}, botId = ${documentCompressorConfig.botId}]")
+        return saveDocumentCompressorConfiguration(documentCompressorConfig)
+    }
+
+    /**
+     * Save the Document Compressor Configuration
+     * @param documentCompressorConfiguration [BotDocumentCompressorConfigurationDTO]
+     */
+    private fun saveDocumentCompressorConfiguration(
+        documentCompressorConfiguration: BotDocumentCompressorConfigurationDTO
+    ): BotDocumentCompressorConfiguration {
+        val documentCompressorConfig = documentCompressorConfiguration.toBotDocumentCompressorConfiguration()
+
+        // Check validity of the document compressor configuration
+        if(documentCompressorConfig.enabled) {
+            DocumentCompressorValidationService.validate(documentCompressorConfig).let { errors ->
+                if (errors.isNotEmpty()) {
+                    throw BadRequestException(errors)
+                }
+            }
+        }
+
+        return try {
+            documentCompressorConfigurationDAO.save(documentCompressorConfig)
+        } catch (e: MongoWriteException) {
+            throw BadRequestException(e.message ?: "Document Compressor Configuration: registration failed on mongo ")
+        } catch (e: Exception) {
+            throw BadRequestException(e.message ?: "Document Compressor Configuration: registration failed ")
+        }
+    }
+
+}

bot/admin/server/src/main/kotlin/service/DocumentCompressorValidationService.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2017/2021 e-voyageurs technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.service
+
+import ai.tock.bot.admin.bot.compressor.BotDocumentCompressorConfiguration
+import ai.tock.genai.orchestratorclient.requests.DocumentCompressorProviderSettingStatusQuery
+import ai.tock.genai.orchestratorclient.responses.ProviderSettingStatusResponse
+import ai.tock.genai.orchestratorclient.services.DocumentCompressorProviderService
+import ai.tock.shared.exception.error.ErrorMessage
+import ai.tock.shared.injector
+import ai.tock.shared.provide
+
+
+object DocumentCompressorValidationService {
+
+    private val documentCompressorProviderService: DocumentCompressorProviderService get() = injector.provide()
+
+    fun validate(config: BotDocumentCompressorConfiguration): Set<ErrorMessage> {
+        return mutableSetOf<ErrorMessage>().apply {
+            addAll(
+                documentCompressorProviderService
+                    .checkSetting(
+                        DocumentCompressorProviderSettingStatusQuery(
+                            setting = config.setting
+                        )
+                    )
+                    .getErrors("Document Compressor setting check failed")
+            )
+        }
+    }
+
+    private fun ProviderSettingStatusResponse?.getErrors(message: String): Set<ErrorMessage> =
+        this?.errors?.map { ErrorMessage(message = message, params = errors) }?.toSet() ?: emptySet()
+
+}

bot/admin/server/src/main/kotlin/service/ObservabilityValidationService.kt

@@ -29,11 +29,11 @@ object ObservabilityValidationService {
 
     private val observabilityProviderService: ObservabilityProviderService get() = injector.provide()
 
-    fun validate(ragConfig: BotObservabilityConfiguration): Set<ErrorMessage> {
+    fun validate(config: BotObservabilityConfiguration): Set<ErrorMessage> {
         return mutableSetOf<ErrorMessage>().apply {
             addAll(
                 observabilityProviderService
-                    .checkSetting(ObservabilityProviderSettingStatusQuery(ragConfig.setting))
+                    .checkSetting(ObservabilityProviderSettingStatusQuery(config.setting))
                     .getErrors("Observability setting check failed")
             )
         }

bot/connector-rest-client/src/main/kotlin/model/ClientFootnote.kt

@@ -23,5 +23,6 @@ data class ClientFootnote(
     val identifier: String,
     val title: String,
     val url: String?,
-    val content: String?
+    val content: String?,
+    val score: Float?
 )

bot/connector-web-model/src/main/kotlin/ai/tock/bot/connector/web/send/Footnote.kt

@@ -37,4 +37,8 @@ data class Footnote(
      * A footnote content
      */
     val content: String?,
+    /**
+     * A footnote score
+     */
+    val score: Float?,
 )

bot/connector-web/src/main/kotlin/WebMessageProcessor.kt

@@ -43,7 +43,8 @@ internal class WebMessageProcessor(private val processMarkdown: Boolean) {
                             footnote.identifier,
                             footnote.title,
                             footnote.url,
-                            footnote.content?.let { postProcess(it) }
+                            footnote.content?.let { postProcess(it) },
+                            footnote.score
                         )
                     })
             }

bot/connector-web/src/test/kotlin/WebConnectorResponseTest.kt

@@ -51,8 +51,8 @@ internal class WebConnectorResponseTest {
             responses = listOf(
                 WebMessageContent(
                     text = "Text with 2 footnotes", footnotes = listOf(
-                        Footnote("e122e97a5cc7", "title 1", url = "https://doc.tock.ai", content = "content 1"),
-                        Footnote("fcad492fdb99", "title 2", url = "https://github.com/theopenconversationkit/tock", content = "content 2")
+                        Footnote("e122e97a5cc7", "title 1", url = "https://doc.tock.ai", content = "content 1", score = 0.965521F),
+                        Footnote("fcad492fdb99", "title 2", url = "https://github.com/theopenconversationkit/tock", content = "content 2", score = 0.965521F)
                     )
                 )
             )

bot/engine/src/main/kotlin/admin/bot/compressor/BotDocumentCompressorConfiguration.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2017/2021 e-voyageurs technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.bot.compressor
+
+import ai.tock.genai.orchestratorcore.models.compressor.DocumentCompressorSetting
+import org.litote.kmongo.Id
+
+data class BotDocumentCompressorConfiguration(
+    val _id: Id<BotDocumentCompressorConfiguration>,
+    val namespace: String,
+    val botId: String,
+    val enabled: Boolean,
+    val setting: DocumentCompressorSetting,
+)

bot/engine/src/main/kotlin/admin/bot/compressor/BotDocumentCompressorConfigurationDAO.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2017/2021 e-voyageurs technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.bot.compressor
+
+import org.litote.kmongo.Id
+
+interface BotDocumentCompressorConfigurationDAO {
+
+    fun listenChanges(listener: () -> Unit)
+
+    fun save(conf: BotDocumentCompressorConfiguration): BotDocumentCompressorConfiguration
+
+    fun findByNamespaceAndBotId(namespace: String, botId: String): BotDocumentCompressorConfiguration?
+
+    fun findByNamespaceAndBotIdAndEnabled(namespace: String, botId: String, enabled: Boolean): BotDocumentCompressorConfiguration?
+
+    fun delete(id: Id<BotDocumentCompressorConfiguration>)
+}