diff --git a/flake.nix b/flake.nix index b119a6f3..02197326 100644 --- a/flake.nix +++ b/flake.nix @@ -91,6 +91,7 @@ solr-delete-core solr-recreate-core solr-recreate-dbtests-cores + solr-logs ]; }; vm = pkgs.mkShell { @@ -119,6 +120,7 @@ vm-solr-create-core vm-solr-delete-core vm-solr-recreate-core + vm-solr-logs solr-recreate-dbtests-cores ]; }; diff --git a/modules/commons/src/main/scala/io/renku/logging/LoggingSetup.scala b/modules/commons/src/main/scala/io/renku/logging/LoggingSetup.scala index 702f00d3..65645810 100644 --- a/modules/commons/src/main/scala/io/renku/logging/LoggingSetup.scala +++ b/modules/commons/src/main/scala/io/renku/logging/LoggingSetup.scala @@ -53,7 +53,7 @@ object LoggingSetup: private def configureRenkuSearch(level: Level): Unit = { scribe - .Logger("io.renku.search") + .Logger("io.renku") .clearHandlers() .withHandler( formatter = Formatter.classic, diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala index 1d6c3719..347c830a 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala @@ -21,6 +21,7 @@ package io.renku.search.api.data import io.bullet.borer.Encoder import io.bullet.borer.derivation.MapBasedCodecs import io.bullet.borer.Decoder +import sttp.tapir.Schema final case class PageDef( limit: Int, @@ -40,3 +41,4 @@ object PageDef: given Encoder[PageDef] = MapBasedCodecs.deriveEncoder given Decoder[PageDef] = MapBasedCodecs.deriveDecoder + given Schema[PageDef] = Schema.derived diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala index 6c6c2747..59b98d1a 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala @@ -18,20 +18,29 @@ package io.renku.search.api.data +import io.bullet.borer.{Decoder, Encoder} +import io.bullet.borer.NullOptions.given +import io.bullet.borer.derivation.MapBasedCodecs +import sttp.tapir.Schema + final case class PageWithTotals( page: PageDef, - prevPage: Option[Int], - nextPage: Option[Int], totalResult: Long, - totalPages: Int + totalPages: Int, + prevPage: Option[Int] = None, + nextPage: Option[Int] = None ) object PageWithTotals: + given Encoder[PageWithTotals] = MapBasedCodecs.deriveEncoder + given Decoder[PageWithTotals] = MapBasedCodecs.deriveDecoder + given Schema[PageWithTotals] = Schema.derived + def apply(page: PageDef, totalResults: Long, hasMore: Boolean): PageWithTotals = PageWithTotals( page, - Option(page.page - 1).filter(_ > 0), - Option(page.page + 1).filter(_ => hasMore), totalResults, - math.ceil(totalResults.toDouble / page.limit).toInt + math.ceil(totalResults.toDouble / page.limit).toInt, + Option(page.page - 1).filter(_ > 0), + Option(page.page + 1).filter(_ => hasMore) ) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala index 7e9522a4..06b71ae0 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala @@ -18,7 +18,17 @@ package io.renku.search.api.data +import io.bullet.borer.derivation.MapBasedCodecs +import io.bullet.borer.Encoder +import io.bullet.borer.Decoder +import sttp.tapir.Schema + final case class SearchResult( items: Seq[SearchEntity], pagingInfo: PageWithTotals ) + +object SearchResult: + given Encoder[SearchResult] = MapBasedCodecs.deriveEncoder + given Decoder[SearchResult] = MapBasedCodecs.deriveDecoder + given Schema[SearchResult] = Schema.derived diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala index 7f4c62bb..632db1b9 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala @@ -58,12 +58,12 @@ object Params extends TapirCodecs with TapirBorerJson { val total: EndpointOutput[Long] = header[Long]("x-total") val totalPages: EndpointOutput[Int] = header[Int]("x-total-pages") - pageDef.and(prevPage).and(nextPage).and(total).and(totalPages).mapTo[PageWithTotals] + pageDef.and(total).and(totalPages).and(prevPage).and(nextPage).mapTo[PageWithTotals] } val searchItems: EndpointOutput[Seq[SearchEntity]] = borerJsonBody[Seq[SearchEntity]] val searchResult: EndpointOutput[SearchResult] = - searchItems.and(pagingInfo).mapTo[SearchResult] + borerJsonBody[SearchResult].and(pagingInfo).map(_._1)(r => (r, r.pagingInfo)) } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala index d2fd99a8..ff058ce8 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala @@ -27,5 +27,4 @@ trait TapirCodecs: Codec.string.mapEither(Query.parse(_))(_.render) given Schema[Query] = Schema.anyObject[Query] - given Schema[PageDef] = Schema.derived given Schema[QueryInput] = Schema.derived diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 40bd8cd3..4812ad31 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -47,7 +47,8 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) res <- solrClient .query[Project]( QueryData(QueryString(solrQuery.query.value, limit, offset)) - .copy(sort = solrQuery.sort) + .withSort(solrQuery.sort) + .withScore ) } yield res diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala index ef61c856..f306833a 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala @@ -45,20 +45,14 @@ final case class QueryData( ) ) + def withSort(sort: SolrSort): QueryData = copy(sort = sort) + def withFieldList(fl: String): QueryData = copy(params = params.updated("fl", fl)) + def withScore: QueryData = withFieldList("* score") + def withScoreAndChildren: QueryData = withFieldList("* score,[child]") + object QueryData: def apply(query: QueryString): QueryData = QueryData(query.q, Nil, query.limit, query.offset, Nil, SolrSort.empty, Map.empty) - def withChildren(query: QueryString): QueryData = - QueryData( - query.q, - Nil, - query.limit, - query.offset, - Nil, - SolrSort.empty, - Map("fl" -> "* score,[child]") - ) - given Encoder[QueryData] = deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/migration/SchemaMigrator.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/migration/SchemaMigrator.scala index 99d9e628..ef9e68ab 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/migration/SchemaMigrator.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/migration/SchemaMigrator.scala @@ -53,10 +53,13 @@ object SchemaMigrator: override def migrate(migrations: Seq[SchemaMigration]): F[Unit] = for { current <- currentVersion + _ <- logger.info(s"Found current schema version '$current' using id $versionDocId") _ <- current.fold(initVersionDocument)(_ => ().pure[F]) remain = migrations.sortBy(_.version).dropWhile(m => current.exists(_ >= m.version)) + _ <- logger.info(s"There are ${remain.size} migrations to run") _ <- remain.traverse_(m => - client.modifySchema(m.commands) >> upsertVersion(m.version) + logger.info(s"Run migration ${m.version}") >> + client.modifySchema(m.commands) >> upsertVersion(m.version) ) } yield () diff --git a/nix/dev-scripts.nix b/nix/dev-scripts.nix index dd56333d..2da65611 100644 --- a/nix/dev-scripts.nix +++ b/nix/dev-scripts.nix @@ -1,7 +1,7 @@ { concatTextFile, writeShellScriptBin, -}: rec { +}: let key = ./dev-vm-key; in rec { redis-push = concatTextFile { name = "redis-push"; files = [./scripts/redis-push]; @@ -37,6 +37,10 @@ ${solr-create-core}/bin/solr-create-core "$1" ''; + solr-logs = writeShellScriptBin "solr-logs" '' + sudo nixos-container run ''${RS_CONTAINER:-rsdev} -- journalctl -efu solr.service + ''; + vm-build = writeShellScriptBin "vm-build" '' nix build .#nixosConfigurations.dev-vm.config.system.build.vm ''; @@ -46,7 +50,11 @@ ''; vm-ssh = writeShellScriptBin "vm-ssh" '' - ssh -p $VM_SSH_PORT root@localhost "$@" + ssh -i ${key} -p $VM_SSH_PORT root@localhost "$@" + ''; + + vm-solr-logs = writeShellScriptBin "solr-logs" '' + ${vm-ssh}/bin/vm-ssh journalctl -efu solr.service ''; vm-solr-create-core = writeShellScriptBin "solr-create-core" '' diff --git a/nix/dev-vm-key b/nix/dev-vm-key new file mode 100644 index 00000000..518517a5 --- /dev/null +++ b/nix/dev-vm-key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHwAAAJDN6ZQezemU +HgAAAAtzc2gtZWQyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHw +AAAEBoaSefL4ulXiGquSLqHHQ9rj+aZZ+YffV49VwEwrduBRI4MbS3TNUcArAT8JKWYKk2 +VH9eJYFywVcD2AMEwJ8fAAAACmVpa2VAcG9yb3MBAgM= +-----END OPENSSH PRIVATE KEY----- diff --git a/nix/dev-vm-key.pub b/nix/dev-vm-key.pub new file mode 100644 index 00000000..13f1d2b4 --- /dev/null +++ b/nix/dev-vm-key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBI4MbS3TNUcArAT8JKWYKk2VH9eJYFywVcD2AMEwJ8f diff --git a/nix/dev-vm.nix b/nix/dev-vm.nix index d69c2219..b7941911 100644 --- a/nix/dev-vm.nix +++ b/nix/dev-vm.nix @@ -17,6 +17,7 @@ users.users.root = { password = "root"; + openssh.authorizedKeys.keyFiles = [ ./dev-vm-key.pub ]; }; i18n = {defaultLocale = "de_DE.UTF-8";}; console.keyMap = "de"; diff --git a/nix/openapi-doc.nix b/nix/openapi-doc.nix index c0312830..e26b7bfe 100644 --- a/nix/openapi-doc.nix +++ b/nix/openapi-doc.nix @@ -20,7 +20,7 @@