From 393ca7ad8ef7957e5afd1449333ff93e52e29b58 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 22 Feb 2024 15:17:36 +0100 Subject: [PATCH] Make query documentation --- build.sbt | 15 +- flake.nix | 1 + .../io/renku/search/api/HttpApplication.scala | 20 ++- .../io/renku/search/api/TapirCodecs.scala | 3 + modules/search-query-docs/docs/manual.md | 161 ++++++++++++++++++ .../search/query/docs/SearchQueryManual.scala | 8 + .../scala/io/renku/search/query/Field.scala | 2 +- project/SearchQueryDocsPlugin.scala | 56 ++++++ 8 files changed, 261 insertions(+), 5 deletions(-) create mode 100644 modules/search-query-docs/docs/manual.md create mode 100644 modules/search-query-docs/src/main/scala/io/renku/search/query/docs/SearchQueryManual.scala create mode 100644 project/SearchQueryDocsPlugin.scala diff --git a/build.sbt b/build.sbt index 8f94e107..c8fbf8cd 100644 --- a/build.sbt +++ b/build.sbt @@ -275,6 +275,18 @@ lazy val searchQuery = project ) .enablePlugins(AutomateHeaderPlugin) +lazy val searchQueryDocs = project + .in(file("modules/search-query-docs")) + .withId("search-query-docs") + .dependsOn(searchQuery) + .enablePlugins(SearchQueryDocsPlugin) + .settings( + name := "search-query-docs", + publish := {}, + publishLocal := {}, + publishArtifact := false + ) + lazy val searchProvision = project .in(file("modules/search-provision")) .withId("search-provision") @@ -309,7 +321,8 @@ lazy val searchApi = project commons % "compile->compile;test->test", http4sBorer % "compile->compile;test->test", searchSolrClient % "compile->compile;test->test", - configValues % "compile->compile;test->test" + configValues % "compile->compile;test->test", + searchQueryDocs % "compile->compile;test->test" ) .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin, RevolverPlugin) diff --git a/flake.nix b/flake.nix index 6a5c3691..829036c3 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,7 @@ redis-push recreate-container + start-container solr-create-core solr-delete-core solr-recreate-core diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala index 805705d6..e601e638 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala @@ -34,6 +34,7 @@ import sttp.tapir.docs.openapi.OpenAPIDocsInterpreter import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.http4s.Http4sServerInterpreter import io.renku.search.query.Query +import io.renku.search.query.docs.SearchQueryManual object HttpApplication: def apply[F[_]: Async: Network]( @@ -59,10 +60,11 @@ class HttpApplication[F[_]: Async](searchApi: SearchApi[F]) private lazy val businessEndpoints: List[ServerEndpoint[Any, F]] = List( - searchEndpoint.serverLogic(searchApi.query) + searchEndpointGet.serverLogic(searchApi.query), + searchEndpointPost.serverLogic(searchApi.query) ) - private lazy val searchEndpoint + private lazy val searchEndpointGet : PublicEndpoint[Query, String, List[SearchEntity], Any] = val q = query[Query]("q").description("User defined query e.g. renku") @@ -70,7 +72,19 @@ class HttpApplication[F[_]: Async](searchApi: SearchApi[F]) .in(q) .errorOut(borerJsonBody[String]) .out(borerJsonBody[List[SearchEntity]]) - .description("Search API for searching Renku entities") + .description(SearchQueryManual.markdown) + + private val searchEndpointPost: PublicEndpoint[Query, String, List[SearchEntity], Any] = + endpoint.post + .errorOut(borerJsonBody[String]) + .in( + borerJsonBody[Query] + .example( + Query(Query.Segment.nameIs("proj-name1"), Query.Segment.text("flight sim")) + ) + ) + .out(borerJsonBody[List[SearchEntity]]) + .description(SearchQueryManual.markdown) private lazy val openAPIEndpoint = val docs = OpenAPIDocsInterpreter() diff --git a/modules/search-api/src/main/scala/io/renku/search/api/TapirCodecs.scala b/modules/search-api/src/main/scala/io/renku/search/api/TapirCodecs.scala index 49bbd01a..1e844d46 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/TapirCodecs.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/TapirCodecs.scala @@ -24,3 +24,6 @@ import io.renku.search.query.Query trait TapirCodecs: given Codec[String, Query, CodecFormat.TextPlain] = Codec.string.mapEither(Query.parse(_))(_.render) + + given Schema[Query] = + Schema.string[Query] diff --git a/modules/search-query-docs/docs/manual.md b/modules/search-query-docs/docs/manual.md new file mode 100644 index 00000000..fa27f744 --- /dev/null +++ b/modules/search-query-docs/docs/manual.md @@ -0,0 +1,161 @@ +## Search Query + +**NOTE: this is a work in progress** + +The search accepts queries in two representations: JSON and a simple +query string. A query may contain specific and unspecific search +terms. + +### Query String + +A query is a sequence of words. All words that are not recognized as +specific search terms are used for searching in various entity +properties, such as `name` or `description`. Specific search terms are +matched exactly against a certain field. Terms are separated by +whitespace. + +Example: +``` +numpy flight visibility:public +``` + +Searches for entities containing `numpy` _and_ `flight` that are +public. + +The term order is usually not relevant, it may influence the score of +a result, though. + +If a value for a specific field contains whitespace, quotes or a comma +it must be enclosed in quotes. Additionally, multiple values can be +provided for each field by using a comma separated list. The values +are treated as alternatives, so any such value would yield a result. + +Example: +``` +numpy flight visibility:public,private +``` + +Searches for entities containing `numpy` _and_ `flight` that are +_either_ `public` _or_ `private`. + +### Query JSON + +The JSON format allows to specify the same query as a JSON object. A +JSON object may contain specific terms by including the corresponding +field-value pair. For unspecific terms, the special field `_text` is +used. + +Example: +```json +{ + "_text": "numpy flight", + "visibility": "public" +} +``` + +JSON objects are sequences of key-value pairs. As such, the encoding +allows to specifiy multiple same named fields in one JSON object. This +would be a valid query: + +```json +{ + "_text": "numpy", + "visibility": "public", + "_text": "flight" +} +``` + +The JSON variant follows the same rules for specifying field values. +Multiple alternative values can be given as a comma separated list. + +### Fields + +The following fields are available: + +```scala mdoc:passthrough +import io.renku.search.query.* +println(Field.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "")) +``` + +Each field allows to specify one or more values, separated by comma. +The value must be separated by a `:`. For date fields, additional `<` +and `>` is supported. + +### Dates + +Date fields, like + +```scala mdoc:passthrough +println(List(Field.Created).map(e => s"`${e.name}`").mkString("- ", "\n- ", "")) +``` + +accept date strings which can be specified in various ways. There are + +- relative dates: `today` +- partial timestamps: `2023-05`, `2023-11-12T10` +- calculations based on the above: `today-5d`, `2023-10-15/10d` + + +#### Relative dates + +There are the following keywords for relative dates: + +```scala mdoc:passthrough +println( + RelativeDate.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "") +) +``` + +#### Partial Timestamps + +Timestamps must be in ISO8601 form and are UTC based and allow to +specify time up to seconds. The full form is + +``` +yyyy-mm-ddTHH:MM:ssZ +``` + +Any part starting from right can be omitted. When querying, it will be +filled with either the maximum or minimum possible value depending on +the side of comparison. When the date is an upper bound, the missing +parts will be set to their minimum values. Conversely, when used as a +lower bound then the parts are set to its maximum value. + +Example: +- `created>2023-03` will turn into `created>2023-03-31T23:59:59` +- `created<2023-03` will turn into `created<2023-03-01T00:00:00` + +#### Date calculations + +At last, a date can be specified by adding or subtracting days from a +reference date. The reference date must be given either as a relative +date or partial timestamp. Then a `+`, `-` or `/` follows with the +amount of days. + +The `/` character allows to add and substract the days from the +reference date, making the reference date the middle. + +Example: +- `created>today-14d` things created from 14 days ago +- `created<2023-05/14d` things created from last two weeks of April + and first two weeks of May + +#### Date Comparison + +Comparing dates with `>` and `<` is done as expected. More interesting +is to specify more than one date and the use of the `:` comparison. + +The `:` can be used to specify ranges more succinctly. For a full +timestamp, it means /equals/. With partial timestamps it searches +within the minimum and maximum possible date for that partial +timestamp. + +Since multiple values are compined using `OR`, it is possible to +search in multiple ranges. + +Example: +``` +created:2023-03,2023-06 +``` + +The above means to match entities created in March 2023 or June 2023. diff --git a/modules/search-query-docs/src/main/scala/io/renku/search/query/docs/SearchQueryManual.scala b/modules/search-query-docs/src/main/scala/io/renku/search/query/docs/SearchQueryManual.scala new file mode 100644 index 00000000..861cebe5 --- /dev/null +++ b/modules/search-query-docs/src/main/scala/io/renku/search/query/docs/SearchQueryManual.scala @@ -0,0 +1,8 @@ +package io.renku.search.query.docs + +object SearchQueryManual { + + lazy val markdown: String = + scala.io.Source.fromURL(getClass.getResource("/query-manual/manual.md")).mkString + +} diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala index 2af7b75f..9679d3ae 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala @@ -34,7 +34,7 @@ object Field: given Encoder[Field] = Encoder.forString.contramap(_.name) given Decoder[Field] = Decoder.forString.mapEither(fromString) - private[this] val allNames: String = Field.values.mkString(", ") + private[this] val allNames: String = Field.values.map(_.name).mkString(", ") def fromString(str: String): Either[String, Field] = Field.values diff --git a/project/SearchQueryDocsPlugin.scala b/project/SearchQueryDocsPlugin.scala new file mode 100644 index 00000000..0c0659b0 --- /dev/null +++ b/project/SearchQueryDocsPlugin.scala @@ -0,0 +1,56 @@ +import sbt._ +import java.nio.file.Path + +object SearchQueryDocsPlugin extends AutoPlugin { + + object autoImport { + val Docs = config("docs") + + val docDirectory = settingKey[File]("The directory containing doc sources") + val outputDirectory = settingKey[File]("The directory to place processed files") + val makeManualFile = taskKey[Unit]("Generate doc file") + + } + import autoImport._ + + override def projectConfigurations: Seq[Configuration] = + Seq(Docs) + + override def projectSettings = + inConfig(Docs)(Defaults.configSettings) ++ Seq( + docDirectory := (Compile / Keys.baseDirectory).value / "docs", + outputDirectory := (Compile / Keys.resourceManaged).value / "query-manual", + Keys.libraryDependencies ++= Seq( + "org.scalameta" %% "mdoc" % "2.5.2" % Docs + ), + makeManualFile := Def.taskDyn { + val cp = (Compile / Keys.dependencyClasspath).value + val cpArg = cp.files.mkString(java.io.File.pathSeparator) + val in = docDirectory.value + val out = outputDirectory.value + IO.createDirectory(out) + + val options = List( + // "--verbose", + "--classpath", + cpArg, + "--in", + in, + "--out", + out + ).mkString(" ") + + (Docs / Keys.runMain).toTask(s" mdoc.SbtMain $options") + }.value, + Compile / Keys.resourceGenerators += Def.task { + val _ = makeManualFile.value + val out = outputDirectory.value + (out ** "*.md").get + }, + Keys.watchSources += Watched.WatchSource( + docDirectory.value, + FileFilter.globFilter("*.md"), + HiddenFileFilter + ) + ) +}