From 744ca94e3f72865a06f43cf95a3a0f8558bdc435 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 13 Feb 2024 18:20:15 +0100 Subject: [PATCH 1/4] WIP: startin with search query --- build.sbt | 15 ++ .../scala/io/renku/commons/Visibility.scala | 25 +++ .../io/renku/search/query/Comparison.scala | 38 ++++ .../renku/search/query/DateTimeParser.scala | 110 ++++++++++++ .../scala/io/renku/search/query/Field.scala | 42 +++++ .../io/renku/search/query/FieldTerm.scala | 61 +++++++ .../scala/io/renku/search/query/Query.scala | 83 +++++++++ .../renku/search/query/QueryJsonCodec.scala | 168 ++++++++++++++++++ .../search/query/DateTimeParserSpec.scala | 38 ++++ .../io/renku/search/query/QueryJsonSpec.scala | 32 ++++ project/Dependencies.scala | 5 + 11 files changed, 617 insertions(+) create mode 100644 modules/commons/src/main/scala/io/renku/commons/Visibility.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/Field.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/Query.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala diff --git a/build.sbt b/build.sbt index b9260f01..b63d12cc 100644 --- a/build.sbt +++ b/build.sbt @@ -57,6 +57,7 @@ lazy val root = project events, redisClient, solrClient, + searchQuery, searchSolrClient, searchProvision, searchApi @@ -233,6 +234,20 @@ lazy val configValues = project searchSolrClient % "compile->compile;test->test" ) +lazy val searchQuery = project + .in(file("modules/search-query")) + .withId("search-query") + .settings(commonSettings) + .settings( + name := "search-query", + libraryDependencies ++= Dependencies.catsParse ++ + Dependencies.borer + ) + .dependsOn( + commons % "compile->compile;test->test" + ) + .enablePlugins(AutomateHeaderPlugin) + lazy val searchProvision = project .in(file("modules/search-provision")) .withId("search-provision") diff --git a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala new file mode 100644 index 00000000..81b5ddbb --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.commons + +enum Visibility: + case Public + case Private + + def name: String = productPrefix diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala b/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala new file mode 100644 index 00000000..b9205a43 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import io.bullet.borer.{Decoder, Encoder} + +enum Comparison: + case Is + case LowerThan + case GreaterThan + + private[query] def asString = this match + case Is => ":" + case LowerThan => "<" + case GreaterThan => ">" + +object Comparison: + given Encoder[Comparison] = Encoder.forString.contramap(_.asString) + given Decoder[Comparison] = Decoder.forString.mapEither(fromString) + + private[query] def fromString(str: String): Either[String, Comparison] = + Comparison.values.find(_.asString == str).toRight(s"Invalid comparison: $str") diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala new file mode 100644 index 00000000..6514400c --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala @@ -0,0 +1,110 @@ +package io.renku.search.query + +import cats.parse.{Numbers, Parser as P} + +import java.time.* + +/** Allows parsing partial date-time strings, filling missing parts with either lowest or + * highest possible values. + */ +final class DateTimeParser(zoneId: ZoneId) { + + val colon: P[Unit] = P.char(':') + val dash: P[Unit] = P.char('-') + val T: P[Unit] = P.char('T') + + val zoneOffset: P[ZoneId] = + P.char('Z').as(ZoneOffset.UTC) + + val optZone = + zoneOffset.orElse(P.pure(zoneId)) + + val year: P[Int] = + (Numbers.nonZeroDigit ~ Numbers.digit.repExactlyAs[String](3)).string.map(_.toInt) + + val month: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 1 && n <= 12) + .withContext("Month not in range 1-12") + + val dom: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 1 && n <= 31) + .withContext("Day not in range 1-31") + + val hour: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 0 && n <= 23) + .withContext("Day not in range 0-23") + + val minsec: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 0 && n <= 59) + .withContext("Minute/second not in range 0-59") + + def withMonth(default: Int) = + (dash *> month).orElse(P.pure(default)) + + def withDay(default: Int) = + (dash *> dom).orElse(P.pure(default)) + + val dateMin = + (year ~ withMonth(1) ~ withDay(1)).map { case ((y, m), d) => + LocalDate.of(y, m, d) + } + + val dateMax = + (year ~ withMonth(12) ~ (dash *> dom).?).map { case ((y, m), dopt) => + val ym = YearMonth.of(y, m) + val md = ym.atEndOfMonth().getDayOfMonth + LocalDate.of(y, m, dopt.getOrElse(md)) + } + + def withMinSec(default: Int) = + (colon *> minsec).orElse(P.pure(default)) + + val timeMin = + (hour ~ withMinSec(0) ~ withMinSec(0)) + .map { case ((h, m), s) => + LocalTime.of(h, m, s) + } + + val timeMax = + (hour ~ withMinSec(59) ~ withMinSec(59)) + .map { case ((h, m), s) => + LocalTime.of(h, m, s) + } + + def withTime(p: P[LocalTime], default: LocalTime) = + (T *> p).orElse(P.pure(default)) + + val offsetDateTimeMin: P[ZonedDateTime] = + (dateMin ~ withTime(timeMin, LocalTime.MIDNIGHT) ~ optZone) + .map { case ((ld, lt), zid) => + ZonedDateTime.of(ld, lt, zid) + } + + val offsetDateTimeMax: P[ZonedDateTime] = + (dateMax ~ withTime(timeMax, LocalTime.MIDNIGHT.minusSeconds(1)) ~ optZone) + .map { case ((ld, lt), zid) => + ZonedDateTime.of(ld, lt, zid) + } + + val instantMin: P[Instant] = offsetDateTimeMin.map(_.toInstant) + val instantMax: P[Instant] = offsetDateTimeMax.map(_.toInstant) +} + +object DateTimeParser: + val utc: DateTimeParser = new DateTimeParser(ZoneOffset.UTC) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala new file mode 100644 index 00000000..8495027e --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import io.bullet.borer.{Decoder, Encoder} + +enum Field: + case ProjectId + case Name + case Slug + case Visibility + case Created + case CreatedBy + + def name: String = productPrefix + +object Field: + given Encoder[Field] = Encoder.forString.contramap(_.name) + given Decoder[Field] = Decoder.forString.mapEither(fromString) + + private[this] val allNames: String = Field.values.mkString(", ") + + def fromString(str: String): Either[String, Field] = + Field.values + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid field: $str. Allowed are: $allNames") diff --git a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala new file mode 100644 index 00000000..0c3421ce --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.data.NonEmptyList +import io.renku.commons.Visibility + +import java.time.Instant +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +enum FieldTerm(val field: Field, val cmp: Comparison): + case ProjectIdIs(values: NonEmptyList[String]) + extends FieldTerm(Field.ProjectId, Comparison.Is) + case NameIs(values: NonEmptyList[String]) extends FieldTerm(Field.Name, Comparison.Is) + case SlugIs(values: NonEmptyList[String]) extends FieldTerm(Field.Slug, Comparison.Is) + case VisibilityIs(values: NonEmptyList[Visibility]) + extends FieldTerm(Field.Visibility, Comparison.Is) + case Created(override val cmp: Comparison, value: Instant) + extends FieldTerm(Field.Created, cmp) + case CreatedByIs(values: NonEmptyList[String]) + extends FieldTerm(Field.CreatedBy, Comparison.Is) + + private[query] def asString = + val value = this match + case ProjectIdIs(values) => FieldTerm.nelToString(values) + case NameIs(values) => FieldTerm.nelToString(values) + case SlugIs(values) => FieldTerm.nelToString(values) + case VisibilityIs(values) => + val vis = values.toList.distinct.map(_.name) + vis.mkString(",") + case Created(_, value) => + val truncated = value.truncatedTo(ChronoUnit.SECONDS) + DateTimeFormatter.ISO_INSTANT.format(truncated) + case CreatedByIs(values) => FieldTerm.nelToString(values) + + s"${field.name}${cmp.asString}${value}" + +object FieldTerm: + private def quote(s: String): String = + if (s.exists(c => c.isWhitespace || c == ',')) s"\"$s\"" + else s + + private def nelToString(nel: NonEmptyList[String]): String = + nel.map(quote).toList.mkString(",") diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala new file mode 100644 index 00000000..51df02da --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.data.NonEmptyList +import io.bullet.borer.{Decoder, Encoder} +import io.renku.commons.Visibility +import io.renku.search.query.Query.Segment + +import java.time.Instant +import java.time.temporal.ChronoUnit + +final case class Query( + segments: List[Query.Segment] +): + def asString: String = + segments + .map { + case Query.Segment.Field(v) => v.asString + case Query.Segment.Text(v) => v + } + .mkString(" ") + + def isEmpty: Boolean = segments.isEmpty + +object Query: + given Encoder[Query] = QueryJsonCodec.encoder.contramap(_.segments) + given Decoder[Query] = QueryJsonCodec.decoder.map(Query.apply) + + enum Segment: + case Field(value: FieldTerm) + case Text(value: String) + + object Segment: + def text(phrase: String): Segment = + Segment.Text(phrase) + + def projectIdIs(value: String, more: String*): Segment = + Segment.Field(FieldTerm.ProjectIdIs(NonEmptyList(value, more.toList))) + + def nameIs(value: String, more: String*): Segment = + Segment.Field(FieldTerm.NameIs(NonEmptyList(value, more.toList))) + + def slugIs(value: String, more: String*): Segment = + Segment.Field(FieldTerm.SlugIs(NonEmptyList(value, more.toList))) + + def visibilityIs(value: Visibility, more: Visibility*): Segment = + Segment.Field(FieldTerm.VisibilityIs(NonEmptyList(value, more.toList))) + + def creationDateIs(date: Instant): Segment = + Segment.Field( + FieldTerm.Created(Comparison.Is, date.truncatedTo(ChronoUnit.SECONDS)) + ) + + def creationDateGreater(date: Instant): Segment = + Segment.Field( + FieldTerm.Created(Comparison.GreaterThan, date.truncatedTo(ChronoUnit.SECONDS)) + ) + + def creationDateLower(date: Instant): Segment = + Segment.Field( + FieldTerm.Created(Comparison.LowerThan, date.truncatedTo(ChronoUnit.SECONDS)) + ) + + val empty: Query = Query(Nil) + + def apply(s: Segment*): Query = Query(s.toList) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala new file mode 100644 index 00000000..5e0470a9 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala @@ -0,0 +1,168 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.data.NonEmptyList +import io.bullet.borer.compat.cats.* +import io.bullet.borer.{Decoder, Encoder, Reader, Writer} +import io.renku.commons.Visibility +import io.renku.search.query.FieldTerm.* +import io.renku.search.query.Query.Segment + +import java.time.Instant +import java.time.temporal.ChronoUnit +import scala.collection.mutable.ListBuffer + +/** Use these json encoding to have it more convenient json than the derived version with + * nested objects or discriminator field. + * + * {{{ + * { + * "projectId": "p1", + * "name": "test", + * "_text": "some phrase", + * "creationDate": ["<", "2024-01-29T12:00"] + * } + * }}} + */ +private object QueryJsonCodec: + // temporary + given Decoder[Visibility] = Decoder.forString.map(Visibility.valueOf) + given Encoder[Visibility] = Encoder.forString.contramap(_.name) + given Decoder[Instant] = + Decoder.forString.map(Instant.parse).map(_.truncatedTo(ChronoUnit.SECONDS)) + given Encoder[Instant] = + Encoder.forString.contramap(_.truncatedTo(ChronoUnit.SECONDS).toString) + + private[this] val freeTextField = "_text" + + enum Name: + case FieldName(v: Field) + case TextName + + private given Decoder[Name] = + new Decoder[Name]: + def read(r: Reader): Name = + if (r.tryReadString(freeTextField)) Name.TextName + else Decoder[Field].map(Name.FieldName.apply).read(r) + + private def writeNelValue[T: Encoder](w: Writer, ts: NonEmptyList[T]): w.type = + if (ts.tail.isEmpty) w.write(ts.head) + else w.writeLinearSeq(ts.toList) + + private def writeFieldTermValue(w: Writer, term: FieldTerm): Writer = + term match + case FieldTerm.ProjectIdIs(values) => + writeNelValue(w, values) + + case FieldTerm.NameIs(values) => + writeNelValue(w, values) + + case FieldTerm.SlugIs(values) => + writeNelValue(w, values) + + case FieldTerm.VisibilityIs(values) => + writeNelValue(w, values) + + case FieldTerm.CreatedByIs(values) => + writeNelValue(w, values) + + case FieldTerm.Created(cmp, date) => + Encoder.forTuple[(Comparison, Instant)].write(w, (cmp, date)) + + def encoder: Encoder[List[Segment]] = + new Encoder[List[Segment]] { + def write(w: Writer, values: List[Segment]): w.type = + w.writeMapOpen(values.size) + values.foreach { + case Segment.Text(v) => + w.writeMapMember(freeTextField, v) + case Segment.Field(v) => + w.write(v.field) + writeFieldTermValue(w, v) + } + w.writeMapClose() + } + + private def readNel[T: Decoder](r: Reader): NonEmptyList[T] = + if (r.hasString) NonEmptyList.of(r.read[T]()) + else Decoder[NonEmptyList[T]].read(r) + + private def readTermValue(r: Reader, name: Name): Segment = + name match + case Name.TextName => + Segment.Text(r.readString()) + + case Name.FieldName(Field.ProjectId) => + val values = readNel[String](r) + Segment.Field(ProjectIdIs(values)) + + case Name.FieldName(Field.Name) => + val values = readNel[String](r) + Segment.Field(NameIs(values)) + + case Name.FieldName(Field.Visibility) => + val values = readNel[Visibility](r) + Segment.Field(VisibilityIs(values)) + + case Name.FieldName(Field.Slug) => + val values = readNel[String](r) + Segment.Field(SlugIs(values)) + + case Name.FieldName(Field.CreatedBy) => + val values = readNel[String](r) + Segment.Field(CreatedByIs(values)) + + case Name.FieldName(Field.Created) => + val (cmp, date) = Decoder.forTuple[(Comparison, Instant)].read(r) + Segment.Field(Created(cmp, date)) + + val decoder: Decoder[List[Segment]] = + new Decoder[List[Segment]] { + def read(r: Reader) = { + val buffer = ListBuffer.newBuilder[Segment] + if (r.hasMapHeader) { + val size = r.readMapHeader() + @annotation.tailrec + def loop(remain: Long): Unit = + if (remain > 0) { + val key = r[Name] + val value = readTermValue(r, key) + buffer.addOne(value) + loop(remain - 1) + } + loop(size) + + } else if (r.hasMapStart) { + r.readMapStart() + @annotation.tailrec + def loop(): Unit = + if (r.tryReadBreak()) () + else { + val key = r[Name] + val value = readTermValue(r, key) + buffer.addOne(value) + loop() + } + loop() + } else r.unexpectedDataItem(expected = "Map") + + buffer.result().result() + } + } diff --git a/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala new file mode 100644 index 00000000..f8e59624 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala @@ -0,0 +1,38 @@ +package io.renku.search.query + +import cats.syntax.all.* +import cats.parse.Parser +import munit.FunSuite + +import java.time.Instant +import java.time.temporal.ChronoUnit + +class DateTimeParserSpec extends FunSuite { + + extension [A](self: Parser[A]) + def run(str: String): A = + self.parseAll(str) match + case Left(err) => + Console.err.println(str) + Console.err.println(err.show) + sys.error("parsing failed") + + case Right(v) => v + + test("playing") { + println( + DateTimeParser.utc.instantMin.run("2023-05-10T11") + ) + println( + DateTimeParser.utc.dateMax.run("2024-02") + ) + println( + DateTimeParser.utc.instantMax.run("2023-02") + ) + println( + DateTimeParser.utc.instantMin.run( + Instant.now().truncatedTo(ChronoUnit.SECONDS).toString + ) + ) + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala new file mode 100644 index 00000000..8e2571b5 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala @@ -0,0 +1,32 @@ +package io.renku.search.query + +import io.bullet.borer.Json +import io.renku.search.query.Query.Segment +import munit.FunSuite + +import java.time.Instant + +class QueryJsonSpec extends FunSuite { + + test("playing") { + println(Query.empty.asString) + val q = Query( + Segment.projectIdIs("p1"), + Segment.text("foo bar"), + Segment.nameIs("ai-project-15048"), + Segment.creationDateLower(Instant.now()) + ) + println(q.asString) + val jsonStr = Json.encode(q).toUtf8String + println(jsonStr) + val decoded = Json.decode(jsonStr.getBytes).to[Query].value + println(decoded) + assertEquals(decoded, q) + + val q2 = Query(Segment.projectIdIs("id-2"), Segment.projectIdIs("id-3")) + val q2Json = Json.encode(q2).toUtf8String + assertEquals(q2Json, """{"ProjectId":"id-2","ProjectId":"id-3"}""") + val decodedQ2 = Json.decode(q2Json.getBytes).to[Query].value + println(decodedQ2) + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d5f951fc..4b8b147d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,6 +10,7 @@ object Dependencies { val catsCore = "2.10.0" val catsEffect = "3.5.3" val catsEffectMunit = "1.0.7" + val catsParse = "1.0.0" val ciris = "3.5.0" val fs2 = "3.9.4" val http4s = "0.23.25" @@ -21,6 +22,10 @@ object Dependencies { val tapir = "1.9.9" } + val catsParse = Seq( + "org.typelevel" %% "cats-parse" % V.catsParse + ) + val ciris = Seq( "is.cir" %% "ciris" % V.ciris ) From 238fe39959435f5a05814f0a184f36fa5a4817b6 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 14 Feb 2024 13:42:48 +0100 Subject: [PATCH 2/4] WIP: moving forward with query parse --- build.sbt | 5 +- .../scala/io/renku/commons/Visibility.scala | 6 +- .../io/renku/search/query/Comparison.scala | 3 + .../io/renku/search/query/DateTimeCalc.scala | 44 +++++ .../renku/search/query/DateTimeParser.scala | 110 ------------- .../io/renku/search/query/DateTimeRef.scala | 43 +++++ .../scala/io/renku/search/query/Field.scala | 5 +- .../io/renku/search/query/FieldTerm.scala | 13 +- .../renku/search/query/PartialDateTime.scala | 112 +++++++++++++ .../scala/io/renku/search/query/Query.scala | 39 +++-- .../io/renku/search/query/RelativeDate.scala | 34 ++++ .../scala/io/renku/search/query/Strings.scala | 25 +++ .../query/{ => json}/QueryJsonCodec.scala | 31 ++-- .../search/query/parse/DateTimeParser.scala | 109 +++++++++++++ .../search/query/parse/QueryParser.scala | 104 ++++++++++++ .../search/query/DateTimeParserSpec.scala | 38 ----- .../search/query/PartialDateTimeSpec.scala | 49 ++++++ .../renku/search/query/QueryGenerators.scala | 151 ++++++++++++++++++ .../io/renku/search/query/QueryJsonSpec.scala | 32 ---- .../io/renku/search/query/StringsSpec.scala | 32 ++++ .../search/query/json/QueryJsonSpec.scala | 51 ++++++ .../query/parse/DateTimeParserSpec.scala | 80 ++++++++++ .../search/query/parse/ParserSuite.scala | 39 +++++ .../search/query/parse/QueryParserSpec.scala | 120 ++++++++++++++ project/Dependencies.scala | 5 + 25 files changed, 1056 insertions(+), 224 deletions(-) create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/DateTimeCalc.scala delete mode 100644 modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/PartialDateTime.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/RelativeDate.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/Strings.scala rename modules/search-query/src/main/scala/io/renku/search/query/{ => json}/QueryJsonCodec.scala (88%) create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/parse/DateTimeParser.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala delete mode 100644 modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/PartialDateTimeSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala delete mode 100644 modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/StringsSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/parse/DateTimeParserSpec.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/parse/ParserSuite.scala create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala diff --git a/build.sbt b/build.sbt index b63d12cc..a5e59eb9 100644 --- a/build.sbt +++ b/build.sbt @@ -299,7 +299,7 @@ lazy val commonSettings = Seq( "-language:postfixOps", // enabling postfixes "-deprecation", // Emit warning and location for usages of deprecated APIs. "-encoding", "utf-8", // Specify character encoding used by source files. - "-explaintypes", // Explain type errors in more detail. + //"-explaintypes", // Explain type errors in more detail. "-feature", // Emit warning and location for usages of features that should be imported explicitly. "-unchecked", // Enable additional warnings where generated code depends on assumptions. "-language:higherKinds", // Allow higher-kinded types @@ -316,7 +316,8 @@ lazy val commonSettings = Seq( Dependencies.scribe, libraryDependencies ++= ( Dependencies.catsEffectMunit ++ - Dependencies.scalacheckEffectMunit + Dependencies.scalacheckEffectMunit ++ + Dependencies.catsScalaCheck ).map(_ % Test), // Format: on organizationName := "Swiss Data Science Center (SDSC)", diff --git a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala index 81b5ddbb..4db38aed 100644 --- a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala +++ b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala @@ -22,4 +22,8 @@ enum Visibility: case Public case Private - def name: String = productPrefix + def name: String = productPrefix.toLowerCase + +object Visibility: + def unsafeFromString(s: String): Visibility = + Visibility.valueOf(s.capitalize) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala b/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala index b9205a43..759acfe1 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Comparison.scala @@ -36,3 +36,6 @@ object Comparison: private[query] def fromString(str: String): Either[String, Comparison] = Comparison.values.find(_.asString == str).toRight(s"Invalid comparison: $str") + + private[query] def unsafeFromString(str: String): Comparison = + fromString(str).fold(sys.error, identity) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeCalc.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeCalc.scala new file mode 100644 index 00000000..bd34a0fa --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeCalc.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import java.time.Period + +final case class DateTimeCalc( + ref: PartialDateTime | RelativeDate, + amount: Period, + range: Boolean +): + def asString: String = + val period = amount.getDays.abs + val sep = + if (range) DateTimeCalc.range + else if (amount.isNegative) DateTimeCalc.sub + else DateTimeCalc.add + ref match + case d: PartialDateTime => + s"${d.asString}$sep${period}d" + + case d: RelativeDate => + s"${d.name}$sep${period}d" + +object DateTimeCalc: + private[query] val add: String = "+" + private[query] val sub: String = "-" + private[query] val range: String = "/" diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala deleted file mode 100644 index 6514400c..00000000 --- a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeParser.scala +++ /dev/null @@ -1,110 +0,0 @@ -package io.renku.search.query - -import cats.parse.{Numbers, Parser as P} - -import java.time.* - -/** Allows parsing partial date-time strings, filling missing parts with either lowest or - * highest possible values. - */ -final class DateTimeParser(zoneId: ZoneId) { - - val colon: P[Unit] = P.char(':') - val dash: P[Unit] = P.char('-') - val T: P[Unit] = P.char('T') - - val zoneOffset: P[ZoneId] = - P.char('Z').as(ZoneOffset.UTC) - - val optZone = - zoneOffset.orElse(P.pure(zoneId)) - - val year: P[Int] = - (Numbers.nonZeroDigit ~ Numbers.digit.repExactlyAs[String](3)).string.map(_.toInt) - - val month: P[Int] = - Numbers.digit - .rep(1, 2) - .string - .map(_.toInt) - .filter(n => n >= 1 && n <= 12) - .withContext("Month not in range 1-12") - - val dom: P[Int] = - Numbers.digit - .rep(1, 2) - .string - .map(_.toInt) - .filter(n => n >= 1 && n <= 31) - .withContext("Day not in range 1-31") - - val hour: P[Int] = - Numbers.digit - .rep(1, 2) - .string - .map(_.toInt) - .filter(n => n >= 0 && n <= 23) - .withContext("Day not in range 0-23") - - val minsec: P[Int] = - Numbers.digit - .rep(1, 2) - .string - .map(_.toInt) - .filter(n => n >= 0 && n <= 59) - .withContext("Minute/second not in range 0-59") - - def withMonth(default: Int) = - (dash *> month).orElse(P.pure(default)) - - def withDay(default: Int) = - (dash *> dom).orElse(P.pure(default)) - - val dateMin = - (year ~ withMonth(1) ~ withDay(1)).map { case ((y, m), d) => - LocalDate.of(y, m, d) - } - - val dateMax = - (year ~ withMonth(12) ~ (dash *> dom).?).map { case ((y, m), dopt) => - val ym = YearMonth.of(y, m) - val md = ym.atEndOfMonth().getDayOfMonth - LocalDate.of(y, m, dopt.getOrElse(md)) - } - - def withMinSec(default: Int) = - (colon *> minsec).orElse(P.pure(default)) - - val timeMin = - (hour ~ withMinSec(0) ~ withMinSec(0)) - .map { case ((h, m), s) => - LocalTime.of(h, m, s) - } - - val timeMax = - (hour ~ withMinSec(59) ~ withMinSec(59)) - .map { case ((h, m), s) => - LocalTime.of(h, m, s) - } - - def withTime(p: P[LocalTime], default: LocalTime) = - (T *> p).orElse(P.pure(default)) - - val offsetDateTimeMin: P[ZonedDateTime] = - (dateMin ~ withTime(timeMin, LocalTime.MIDNIGHT) ~ optZone) - .map { case ((ld, lt), zid) => - ZonedDateTime.of(ld, lt, zid) - } - - val offsetDateTimeMax: P[ZonedDateTime] = - (dateMax ~ withTime(timeMax, LocalTime.MIDNIGHT.minusSeconds(1)) ~ optZone) - .map { case ((ld, lt), zid) => - ZonedDateTime.of(ld, lt, zid) - } - - val instantMin: P[Instant] = offsetDateTimeMin.map(_.toInstant) - val instantMax: P[Instant] = offsetDateTimeMax.map(_.toInstant) -} - -object DateTimeParser: - val utc: DateTimeParser = new DateTimeParser(ZoneOffset.UTC) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala new file mode 100644 index 00000000..3b9d9387 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.syntax.all.* +import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.query.parse.DateTimeParser + +enum DateTimeRef: + case Literal(ref: PartialDateTime) + case Relative(ref: RelativeDate) + case Calc(ref: DateTimeCalc) + + val asString: String = this match + case Literal(ref) => ref.asString + case Relative(ref) => ref.name + case Calc(ref) => ref.asString + +object DateTimeRef: + given Encoder[DateTimeRef] = Encoder.forString.contramap(_.asString) + given Decoder[DateTimeRef] = Decoder.forString.mapEither { str => + DateTimeParser.dateTimeRef.parseAll(str).leftMap(_.show) + } + + def apply(ref: PartialDateTime): DateTimeRef = Literal(ref) + def apply(ref: RelativeDate): DateTimeRef = Relative(ref) + def apply(ref: DateTimeCalc): DateTimeRef = Calc(ref) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala index 8495027e..2af7b75f 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala @@ -28,7 +28,7 @@ enum Field: case Created case CreatedBy - def name: String = productPrefix + val name: String = Strings.lowerFirst(productPrefix) object Field: given Encoder[Field] = Encoder.forString.contramap(_.name) @@ -40,3 +40,6 @@ object Field: Field.values .find(_.name.equalsIgnoreCase(str)) .toRight(s"Invalid field: $str. Allowed are: $allNames") + + def unsafeFromString(str: String): Field = + fromString(str).fold(sys.error, identity) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala index 0c3421ce..c27f202d 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala @@ -21,10 +21,6 @@ package io.renku.search.query import cats.data.NonEmptyList import io.renku.commons.Visibility -import java.time.Instant -import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit - enum FieldTerm(val field: Field, val cmp: Comparison): case ProjectIdIs(values: NonEmptyList[String]) extends FieldTerm(Field.ProjectId, Comparison.Is) @@ -32,7 +28,7 @@ enum FieldTerm(val field: Field, val cmp: Comparison): case SlugIs(values: NonEmptyList[String]) extends FieldTerm(Field.Slug, Comparison.Is) case VisibilityIs(values: NonEmptyList[Visibility]) extends FieldTerm(Field.Visibility, Comparison.Is) - case Created(override val cmp: Comparison, value: Instant) + case Created(override val cmp: Comparison, values: NonEmptyList[DateTimeRef]) extends FieldTerm(Field.Created, cmp) case CreatedByIs(values: NonEmptyList[String]) extends FieldTerm(Field.CreatedBy, Comparison.Is) @@ -45,16 +41,15 @@ enum FieldTerm(val field: Field, val cmp: Comparison): case VisibilityIs(values) => val vis = values.toList.distinct.map(_.name) vis.mkString(",") - case Created(_, value) => - val truncated = value.truncatedTo(ChronoUnit.SECONDS) - DateTimeFormatter.ISO_INSTANT.format(truncated) + case Created(_, values) => FieldTerm.nelToString(values.map(_.asString)) case CreatedByIs(values) => FieldTerm.nelToString(values) s"${field.name}${cmp.asString}${value}" object FieldTerm: private def quote(s: String): String = - if (s.exists(c => c.isWhitespace || c == ',')) s"\"$s\"" + if (s.exists(c => c.isWhitespace || c == ',' || c == '"')) + s"\"${s.replace("\"", "\\\"")}\"" else s private def nelToString(nel: NonEmptyList[String]): String = diff --git a/modules/search-query/src/main/scala/io/renku/search/query/PartialDateTime.scala b/modules/search-query/src/main/scala/io/renku/search/query/PartialDateTime.scala new file mode 100644 index 00000000..8145d75a --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/PartialDateTime.scala @@ -0,0 +1,112 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.syntax.all.* +import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.query.PartialDateTime.prefixT +import io.renku.search.query.parse.DateTimeParser + +import java.time.* +import java.time.temporal.ChronoUnit + +final case class PartialDateTime( + date: PartialDateTime.Date, + time: Option[PartialDateTime.Time] = None, + zoneId: Option[ZoneId] = None +): + def asString: String = + s"${date.asString}${prefixT(time.map(_.asString))}${zoneId.map(_.getId).orEmpty}" + + def isExact: Boolean = + date.isExact && time.exists(_.isExact) + + def zonedMax(defaultZone: ZoneId): ZonedDateTime = + ZonedDateTime.of( + date.max, + time.map(_.max).getOrElse(PartialDateTime.maxTime), + zoneId.getOrElse(defaultZone) + ) + + def instantMax(defaultZone: ZoneId): Instant = + zonedMax(defaultZone).toInstant.truncatedTo(ChronoUnit.SECONDS) + + def zonedMin(defaultZone: ZoneId): ZonedDateTime = + ZonedDateTime.of( + date.min, + time.map(_.min).getOrElse(LocalTime.MIDNIGHT), + zoneId.getOrElse(defaultZone) + ) + + def instantMin(defaultZone: ZoneId): Instant = + zonedMin(defaultZone).toInstant.truncatedTo(ChronoUnit.SECONDS) + +object PartialDateTime: + private val maxTime: LocalTime = LocalTime.MIDNIGHT.minusSeconds(1) + private def dash(n: Option[Int]): String = + n.map(i => s"-$i").getOrElse("") + private def colon(n: Option[Int]): String = + n.map(i => s":$i").getOrElse("") + private def prefixT(str: Option[String]): String = + str.map(s => s"T$s").getOrElse("") + + given Encoder[PartialDateTime] = Encoder.forString.contramap(_.asString) + given Decoder[PartialDateTime] = Decoder.forString.mapEither { str => + DateTimeParser.partialDateTime.parseAll(str.trim).leftMap(_.show) + } + + def fromInstant(dt: Instant): PartialDateTime = + fromString(dt.truncatedTo(ChronoUnit.SECONDS).toString) + .fold(err => sys.error(s"Parsing valid instant failed\n$err"), identity) + + def fromString(str: String): Either[String, PartialDateTime] = + DateTimeParser.partialDateTime + .parseAll(str) + .leftMap(_.show) + + def unsafeFromString(str: String): PartialDateTime = + fromString(str).fold(sys.error, identity) + + final case class Date( + year: Int, + month: Option[Int] = None, + dayOfMonth: Option[Int] = None + ): + def asString: String = s"$year${dash(month)}${dash(dayOfMonth)}" + def isExact: Boolean = month.isDefined && dayOfMonth.isDefined + def max: LocalDate = { + val m = month.getOrElse(12) + val md = YearMonth.of(year, m).atEndOfMonth().getDayOfMonth + LocalDate.of(year, m, dayOfMonth.getOrElse(md)) + } + def min: LocalDate = + LocalDate.of(year, month.getOrElse(1), dayOfMonth.getOrElse(1)) + + final case class Time( + hour: Int, + minute: Option[Int] = None, + sec: Option[Int] = None + ): + def asString: String = s"$hour${colon(minute)}${colon(sec)}" + def isExact: Boolean = minute.isDefined && sec.isDefined + def max: LocalTime = + LocalTime.of(hour, minute.getOrElse(59), sec.getOrElse(59)) + + def min: LocalTime = + LocalTime.of(hour, minute.getOrElse(0), sec.getOrElse(0)) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index 51df02da..d86f4368 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -19,12 +19,13 @@ package io.renku.search.query import cats.data.NonEmptyList +import cats.syntax.all.* import io.bullet.borer.{Decoder, Encoder} import io.renku.commons.Visibility +import io.renku.search.query.FieldTerm.Created import io.renku.search.query.Query.Segment - -import java.time.Instant -import java.time.temporal.ChronoUnit +import io.renku.search.query.json.QueryJsonCodec +import io.renku.search.query.parse.QueryParser final case class Query( segments: List[Query.Segment] @@ -43,6 +44,11 @@ object Query: given Encoder[Query] = QueryJsonCodec.encoder.contramap(_.segments) given Decoder[Query] = QueryJsonCodec.decoder.map(Query.apply) + def parse(str: String): Either[String, Query] = + val trimmed = str.trim + if (trimmed.isEmpty) Right(empty) + else QueryParser.query.parseAll(trimmed).leftMap(_.show) + enum Segment: case Field(value: FieldTerm) case Text(value: String) @@ -63,20 +69,23 @@ object Query: def visibilityIs(value: Visibility, more: Visibility*): Segment = Segment.Field(FieldTerm.VisibilityIs(NonEmptyList(value, more.toList))) - def creationDateIs(date: Instant): Segment = - Segment.Field( - FieldTerm.Created(Comparison.Is, date.truncatedTo(ChronoUnit.SECONDS)) - ) + def creationDateIs(date: DateTimeRef, dates: DateTimeRef*): Segment = + Segment.Field(Created(Comparison.Is, NonEmptyList(date, dates.toList))) + + def creationDateLt(date: DateTimeRef, dates: DateTimeRef*): Segment = + Segment.Field(Created(Comparison.LowerThan, NonEmptyList(date, dates.toList))) + + def creationDateGt(date: DateTimeRef, dates: DateTimeRef*): Segment = + Segment.Field(Created(Comparison.GreaterThan, NonEmptyList(date, dates.toList))) + + def creationDateIs(date: PartialDateTime, dates: PartialDateTime*): Segment = + creationDateIs(DateTimeRef(date), dates.map(DateTimeRef.apply): _*) - def creationDateGreater(date: Instant): Segment = - Segment.Field( - FieldTerm.Created(Comparison.GreaterThan, date.truncatedTo(ChronoUnit.SECONDS)) - ) + def creationDateGt(date: PartialDateTime, dates: PartialDateTime*): Segment = + creationDateGt(DateTimeRef(date), dates.map(DateTimeRef.apply): _*) - def creationDateLower(date: Instant): Segment = - Segment.Field( - FieldTerm.Created(Comparison.LowerThan, date.truncatedTo(ChronoUnit.SECONDS)) - ) + def creationDateLt(date: PartialDateTime, dates: PartialDateTime*): Segment = + creationDateLt(DateTimeRef(date), dates.map(DateTimeRef.apply): _*) val empty: Query = Query(Nil) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/RelativeDate.scala b/modules/search-query/src/main/scala/io/renku/search/query/RelativeDate.scala new file mode 100644 index 00000000..1704f80e --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/RelativeDate.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +enum RelativeDate: + case Today + case Yesterday + + val name: String = productPrefix.toLowerCase + +object RelativeDate: + def fromString(str: String): Either[String, RelativeDate] = + RelativeDate.values + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid relative date-time: $str") + + def unsafeFromString(str: String): RelativeDate = + fromString(str).fold(sys.error, identity) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Strings.scala b/modules/search-query/src/main/scala/io/renku/search/query/Strings.scala new file mode 100644 index 00000000..49281483 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/Strings.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +private object Strings { + def lowerFirst(s: String) = + if (s == null || s.isEmpty || s.charAt(0).isLower) s + else s.updated(0, s.charAt(0).toLower) +} diff --git a/modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala similarity index 88% rename from modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala rename to modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index 5e0470a9..224d6a05 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -16,39 +16,37 @@ * limitations under the License. */ -package io.renku.search.query +package io.renku.search.query.json import cats.data.NonEmptyList import io.bullet.borer.compat.cats.* import io.bullet.borer.{Decoder, Encoder, Reader, Writer} import io.renku.commons.Visibility +import io.renku.search.query.* import io.renku.search.query.FieldTerm.* import io.renku.search.query.Query.Segment -import java.time.Instant -import java.time.temporal.ChronoUnit import scala.collection.mutable.ListBuffer /** Use these json encoding to have it more convenient json than the derived version with * nested objects or discriminator field. * * {{{ - * { - * "projectId": "p1", - * "name": "test", - * "_text": "some phrase", - * "creationDate": ["<", "2024-01-29T12:00"] - * } + * [ + * { + * "projectId": ["p1", "p2"], + * "projectId": "p2", + * "name": "test", + * "_text": "some phrase", + * "creationDate": ["<", "2024-01-29T12:00"] + * } + * ] * }}} */ -private object QueryJsonCodec: +private[query] object QueryJsonCodec: // temporary given Decoder[Visibility] = Decoder.forString.map(Visibility.valueOf) given Encoder[Visibility] = Encoder.forString.contramap(_.name) - given Decoder[Instant] = - Decoder.forString.map(Instant.parse).map(_.truncatedTo(ChronoUnit.SECONDS)) - given Encoder[Instant] = - Encoder.forString.contramap(_.truncatedTo(ChronoUnit.SECONDS).toString) private[this] val freeTextField = "_text" @@ -84,7 +82,7 @@ private object QueryJsonCodec: writeNelValue(w, values) case FieldTerm.Created(cmp, date) => - Encoder.forTuple[(Comparison, Instant)].write(w, (cmp, date)) + Encoder.forTuple[(Comparison, List[DateTimeRef])].write(w, (cmp, date.toList)) def encoder: Encoder[List[Segment]] = new Encoder[List[Segment]] { @@ -130,7 +128,8 @@ private object QueryJsonCodec: Segment.Field(CreatedByIs(values)) case Name.FieldName(Field.Created) => - val (cmp, date) = Decoder.forTuple[(Comparison, Instant)].read(r) + val (cmp, date) = + Decoder.forTuple[(Comparison, NonEmptyList[DateTimeRef])].read(r) Segment.Field(Created(cmp, date)) val decoder: Decoder[List[Segment]] = diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/DateTimeParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/DateTimeParser.scala new file mode 100644 index 00000000..bfad2ac8 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/DateTimeParser.scala @@ -0,0 +1,109 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import cats.parse.{Numbers, Parser as P} +import io.renku.search.query.{DateTimeCalc, DateTimeRef, PartialDateTime, RelativeDate} + +import java.time.* + +/** Allows parsing partial date-time strings, filling missing parts with either lowest or + * highest possible values. + */ +object DateTimeParser { + + val colon: P[Unit] = P.char(':') + val dash: P[Unit] = P.char('-') + val T: P[Unit] = P.char('T') + + val utcOffset: P[ZoneId] = + P.char('Z').as(ZoneOffset.UTC) + + val year: P[Int] = + (Numbers.nonZeroDigit ~ Numbers.digit.repExactlyAs[String](3)).string.map(_.toInt) + + val month: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 1 && n <= 12) + .withContext("Month not in range 1-12") + + val dom: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 1 && n <= 31) + .withContext("Day not in range 1-31") + + val hour: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 0 && n <= 23) + .withContext("Day not in range 0-23") + + val minsec: P[Int] = + Numbers.digit + .rep(1, 2) + .string + .map(_.toInt) + .filter(n => n >= 0 && n <= 59) + .withContext("Minute/second not in range 0-59") + + val date = (year ~ (dash *> month).? ~ (dash *> dom).?).map { case ((y, m), d) => + PartialDateTime.Date(y, m, d) + } + + val time = (hour ~ (colon *> minsec).? ~ (colon *> minsec).?).map { case ((h, m), s) => + PartialDateTime.Time(h, m, s) + } + + val partialDateTime: P[PartialDateTime] = + (date ~ (T *> time).? ~ utcOffset.?).map { case ((d, t), z) => + PartialDateTime(d, t, z) + } + + val relativeDate: P[RelativeDate] = + P.stringIn(RelativeDate.values.map(_.name).toSeq).map(RelativeDate.unsafeFromString) + + val dateCalc: P[DateTimeCalc] = { + val ref: P[PartialDateTime | RelativeDate] = relativeDate | partialDateTime + val sep = P.stringIn(Seq(DateTimeCalc.add, DateTimeCalc.sub, DateTimeCalc.range)) + val days = + ((Numbers.nonZeroDigit ~ Numbers.digits0).string <* P.charIn("dD").void).map(s => + Period.ofDays(s.toInt) + ) + (ref ~ sep ~ days).map { case ((date, op), amount) => + val p = + if (op == DateTimeCalc.sub) amount.negated() + else amount + DateTimeCalc(date, p, op == DateTimeCalc.range) + } + } + + val dateTimeRef: P[DateTimeRef] = + dateCalc.map(DateTimeRef.apply).backtrack | + partialDateTime.map(DateTimeRef.apply) | + relativeDate.map(DateTimeRef.apply) + +} diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala new file mode 100644 index 00000000..326c3f83 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -0,0 +1,104 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import cats.data.NonEmptyList +import cats.parse.{Parser as P, Parser0 as P0} +import io.renku.commons.Visibility +import io.renku.search.query.* + +private[query] object QueryParser { + val basicString = + P.charsWhile(c => c > ' ' && !c.isWhitespace && c != '"' && c != '\\' && c != ',') + + val qstring = + basicString.backtrack.orElse(cats.parse.strings.Json.delimited.parser) + + val sp0: P0[Unit] = P.charsWhile0(_.isWhitespace).void + val sp: P[Unit] = P.charsWhile(_.isWhitespace).void + val comma: P[Unit] = P.char(',') + val commaSep = comma.surroundedBy(sp0).backtrack + + def mkFieldNames(fs: Set[Field]) = + fs.map(_.name) ++ fs.map(_.name.toLowerCase) + + def fieldNameFrom(candidates: Set[Field]) = + P.stringIn(mkFieldNames(candidates)).map(Field.unsafeFromString) + + val comparison: P[Comparison] = + P.stringIn(Comparison.values.map(_.asString)).map(Comparison.unsafeFromString) + + val is: P[Unit] = P.string(Comparison.Is.asString) + val gt: P[Unit] = P.string(Comparison.GreaterThan.asString) + val lt: P[Unit] = P.string(Comparison.LowerThan.asString) + + val visibility: P[Visibility] = + P.stringIn( + Visibility.values + .map(_.name.toLowerCase) + .toSet ++ Visibility.values.map(_.name).toSet + ).map(Visibility.unsafeFromString) + + def nelOf[A](p: P[A], sep: P[Unit]) = + (p ~ (sep *> p).rep0).map { case (h, t) => NonEmptyList(h, t) } + + val values: P[NonEmptyList[String]] = + nelOf(qstring, commaSep) + + val visibilities: P[NonEmptyList[Visibility]] = + nelOf(visibility, commaSep) + + val termIs: P[FieldTerm] = { + val field = fieldNameFrom(Field.values.toSet - Field.Created - Field.Visibility) + ((field <* is) ~ values).map { case (f, v) => + f match + case Field.Name => FieldTerm.NameIs(v) + case Field.ProjectId => FieldTerm.ProjectIdIs(v) + case Field.Slug => FieldTerm.SlugIs(v) + case Field.CreatedBy => FieldTerm.CreatedByIs(v) + case Field.Visibility => sys.error("visibility not allowed") + case Field.Created => sys.error("created not allowed") + } + } + + val visibilityIs: P[FieldTerm] = { + val field = fieldNameFrom(Set(Field.Visibility)) + ((field ~ is).void *> visibilities).map(v => FieldTerm.VisibilityIs(v)) + } + + val created: P[FieldTerm] = { + val field = fieldNameFrom(Set(Field.Created)) + ((field *> comparison) ~ nelOf(DateTimeParser.dateTimeRef, commaSep)).map { + case (cmp, pdate) => + FieldTerm.Created(cmp, pdate) + } + } + + val fieldTerm: P[FieldTerm] = termIs | visibilityIs | created + + val freeText: P[String] = + P.charsWhile(c => !c.isWhitespace) + + val segment: P[Query.Segment] = + fieldTerm.map(Query.Segment.Field.apply) | + freeText.map(Query.Segment.Text.apply) + + val query: P[Query] = + segment.repSep(min = 1, sp).map(s => Query(s.toList)) +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala deleted file mode 100644 index f8e59624..00000000 --- a/modules/search-query/src/test/scala/io/renku/search/query/DateTimeParserSpec.scala +++ /dev/null @@ -1,38 +0,0 @@ -package io.renku.search.query - -import cats.syntax.all.* -import cats.parse.Parser -import munit.FunSuite - -import java.time.Instant -import java.time.temporal.ChronoUnit - -class DateTimeParserSpec extends FunSuite { - - extension [A](self: Parser[A]) - def run(str: String): A = - self.parseAll(str) match - case Left(err) => - Console.err.println(str) - Console.err.println(err.show) - sys.error("parsing failed") - - case Right(v) => v - - test("playing") { - println( - DateTimeParser.utc.instantMin.run("2023-05-10T11") - ) - println( - DateTimeParser.utc.dateMax.run("2024-02") - ) - println( - DateTimeParser.utc.instantMax.run("2023-02") - ) - println( - DateTimeParser.utc.instantMin.run( - Instant.now().truncatedTo(ChronoUnit.SECONDS).toString - ) - ) - } -} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/PartialDateTimeSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/PartialDateTimeSpec.scala new file mode 100644 index 00000000..caf5aa2b --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/PartialDateTimeSpec.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import munit.FunSuite + +import java.time.{Instant, ZoneOffset} + +class PartialDateTimeSpec extends FunSuite { + val utc = ZoneOffset.UTC + + test("minimum") { + assertEquals( + PartialDateTime.unsafeFromString("2023-01").instantMin(utc), + Instant.parse("2023-01-01T00:00:00Z") + ) + } + + test("leap year") { + assertEquals( + PartialDateTime + .unsafeFromString("2024-02") + .instantMax(utc), + Instant.parse("2024-02-29T23:59:59Z") + ) + assertEquals( + PartialDateTime + .unsafeFromString("2023-02") + .instantMax(utc), + Instant.parse("2023-02-28T23:59:59Z") + ) + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala new file mode 100644 index 00000000..f27c30d9 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -0,0 +1,151 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.data.NonEmptyList +import cats.syntax.all.* +import io.renku.commons.Visibility +import org.scalacheck.Gen +import org.scalacheck.cats.implicits.* + +import java.time.{Period, YearMonth, ZoneId, ZoneOffset} + +object QueryGenerators: + val utc: Gen[Option[ZoneId]] = + Gen.frequency(2 -> Gen.const(Some(ZoneOffset.UTC)), 1 -> Gen.const(None)) + + val partialTime: Gen[PartialDateTime.Time] = + for { + h <- Gen.choose(0, 23) + m <- Gen.option(Gen.choose(0, 59)) + s <- Gen.option(Gen.choose(0, 59)) + } yield PartialDateTime.Time(h, m, s) + + val partialDate: Gen[PartialDateTime.Date] = + for { + y <- Gen.choose(1900, 2200) + m <- Gen.option(Gen.choose(1, 12)) + maxDay = m.map(ym => YearMonth.of(y, ym).atEndOfMonth().getDayOfMonth) + d <- maxDay.traverse(max => Gen.choose(1, max)) + } yield PartialDateTime.Date(y, m, d) + + val partialDateTime: Gen[PartialDateTime] = + (partialDate, Gen.option(partialTime), utc).mapN(PartialDateTime.apply) + + val relativeDate: Gen[RelativeDate] = + Gen.oneOf(RelativeDate.values.toSeq) + + val dateTimeCalc: Gen[DateTimeCalc] = { + val ref: Gen[PartialDateTime | RelativeDate] = + Gen.oneOf(partialDateTime, relativeDate) + + val period: Gen[Period] = + Gen.oneOf((-8 to -1) ++ (1 to 8)).map(n => Period.ofDays(n)) + + for { + date <- ref + amount <- period + range <- Gen.oneOf(true, false) + } yield DateTimeCalc(date, amount, range) + } + + val dateTimeRef: Gen[DateTimeRef] = + Gen.oneOf( + partialDateTime.map(DateTimeRef.apply), + relativeDate.map(DateTimeRef.apply), + dateTimeCalc.map(DateTimeRef.apply) + ) + + val field: Gen[Field] = + Gen.oneOf(Field.values.toSeq) + + // TODO move to commons + val visibility: Gen[Visibility] = + Gen.oneOf(Visibility.values.toSeq) + + // TODO move to commons + def nelOfN[A](n: Int, gen: Gen[A]): Gen[NonEmptyList[A]] = + for { + e0 <- gen + en <- Gen.listOfN(n - 1, gen) + } yield NonEmptyList(e0, en) + + private val simpleString: Gen[String] = Gen.alphaNumStr + private val quotedString: Gen[String] = + Gen.alphaNumStr.map(s => s"\"$s\"") + + private val valueString: Gen[String] = + Gen.oneOf(simpleString, quotedString) + + private val stringValues: Gen[NonEmptyList[String]] = + Gen.choose(1, 4).flatMap(n => nelOfN(n, valueString)) + + val projectIdTerm: Gen[FieldTerm] = + stringValues.map(FieldTerm.ProjectIdIs(_)) + + val nameTerm: Gen[FieldTerm] = + stringValues.map(FieldTerm.NameIs(_)) + + val slugTerm: Gen[FieldTerm] = + stringValues.map(FieldTerm.SlugIs(_)) + + val createdByTerm: Gen[FieldTerm] = + stringValues.map(FieldTerm.CreatedByIs(_)) + + val visibilityTerm: Gen[FieldTerm] = + Gen + .frequency(10 -> visibility.map(NonEmptyList.one), 1 -> nelOfN(2, visibility)) + .map(FieldTerm.VisibilityIs(_)) + + private val comparison: Gen[Comparison] = + Gen.oneOf(Comparison.values.toSeq) + + val createdTerm: Gen[FieldTerm] = + for { + cmp <- comparison + len <- Gen.frequency(5 -> Gen.const(1), 1 -> Gen.choose(1, 3)) + pd <- nelOfN(len, dateTimeRef) + } yield FieldTerm.Created(cmp, pd) + + val fieldTerm: Gen[FieldTerm] = + Gen.oneOf( + projectIdTerm, + nameTerm, + slugTerm, + createdByTerm, + visibilityTerm, + createdTerm + ) + + val freeText: Gen[String] = + Gen.choose(1, 5).flatMap { len => + Gen.listOfN(len, valueString).map(_.mkString(" ")) + } + + val segment: Gen[Query.Segment] = + Gen.oneOf( + fieldTerm.map(Query.Segment.Field.apply), + freeText.map(Query.Segment.Text.apply) + ) + + val query: Gen[Query] = + Gen + .choose(0, 12) + .flatMap(n => Gen.listOfN(n, segment)) + .map(Query.apply) diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala deleted file mode 100644 index 8e2571b5..00000000 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryJsonSpec.scala +++ /dev/null @@ -1,32 +0,0 @@ -package io.renku.search.query - -import io.bullet.borer.Json -import io.renku.search.query.Query.Segment -import munit.FunSuite - -import java.time.Instant - -class QueryJsonSpec extends FunSuite { - - test("playing") { - println(Query.empty.asString) - val q = Query( - Segment.projectIdIs("p1"), - Segment.text("foo bar"), - Segment.nameIs("ai-project-15048"), - Segment.creationDateLower(Instant.now()) - ) - println(q.asString) - val jsonStr = Json.encode(q).toUtf8String - println(jsonStr) - val decoded = Json.decode(jsonStr.getBytes).to[Query].value - println(decoded) - assertEquals(decoded, q) - - val q2 = Query(Segment.projectIdIs("id-2"), Segment.projectIdIs("id-3")) - val q2Json = Json.encode(q2).toUtf8String - assertEquals(q2Json, """{"ProjectId":"id-2","ProjectId":"id-3"}""") - val decodedQ2 = Json.decode(q2Json.getBytes).to[Query].value - println(decodedQ2) - } -} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/StringsSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/StringsSpec.scala new file mode 100644 index 00000000..6acbf63b --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/StringsSpec.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import munit.FunSuite + +class StringsSpec extends FunSuite { + + extension (s: String) def lowerFirst = Strings.lowerFirst(s) + + test("make lower") { + assertEquals("Cap".lowerFirst, "cap") + assertEquals("cup".lowerFirst, "cup") + assertEquals("ProductId".lowerFirst, "productId") + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala new file mode 100644 index 00000000..79d791f4 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.json + +import io.bullet.borer.Json +import io.renku.search.query.{PartialDateTime, Query} +import io.renku.search.query.Query.Segment +import munit.FunSuite + +import java.time.Instant + +class QueryJsonSpec extends FunSuite { + + test("playing") { + println(Query.empty.asString) + val q = Query( + Segment.projectIdIs("p1"), + Segment.text("foo bar"), + Segment.nameIs("ai-project-15048"), + Segment.creationDateLt(PartialDateTime.fromInstant(Instant.now())) + ) + println(q.asString) + val jsonStr = Json.encode(q).toUtf8String + println(jsonStr) + val decoded = Json.decode(jsonStr.getBytes).to[Query].value + println(decoded) + assertEquals(decoded, q) + + val q2 = Query(Segment.projectIdIs("id-2"), Segment.projectIdIs("id-3")) + val q2Json = Json.encode(q2).toUtf8String + assertEquals(q2Json, """{"projectId":"id-2","projectId":"id-3"}""") + val decodedQ2 = Json.decode(q2Json.getBytes).to[Query].value + println(decodedQ2) + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/DateTimeParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/DateTimeParserSpec.scala new file mode 100644 index 00000000..0ecdad3d --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/DateTimeParserSpec.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import io.renku.search.query.{DateTimeCalc, PartialDateTime} +import munit.FunSuite + +import java.time.{Instant, Period, ZoneOffset} +import java.time.temporal.ChronoUnit + +class DateTimeParserSpec extends FunSuite with ParserSuite { + val utc = ZoneOffset.UTC + + test("PartialDate: current time") { + val now = Instant.now().truncatedTo(ChronoUnit.SECONDS) + assertEquals( + DateTimeParser.partialDateTime.run(now.toString).instantMin(utc), + now + ) + assertEquals( + DateTimeParser.partialDateTime.run(now.toString).instantMax(utc), + now + ) + } + + test("PartialDate: no time") { + assertEquals( + DateTimeParser.partialDateTime.run("2024-02"), + PartialDateTime(PartialDateTime.Date(2024, Some(2))) + ) + assertEquals( + DateTimeParser.partialDateTime.run("2023-02-14"), + PartialDateTime(PartialDateTime.Date(2023, Some(2), Some(14))) + ) + } + + test("DateCalc") { + val p = DateTimeParser.dateCalc + assertEquals( + p.run("2023-02-15-10d"), + DateTimeCalc( + PartialDateTime.unsafeFromString("2023-02-15"), + Period.ofDays(-10), + false + ) + ) + assertEquals( + p.run("2023-02+10d"), + DateTimeCalc( + PartialDateTime.unsafeFromString("2023-02"), + Period.ofDays(10), + false + ) + ) + assertEquals( + p.run("2023-02/10d"), + DateTimeCalc( + PartialDateTime.unsafeFromString("2023-02"), + Period.ofDays(10), + true + ) + ) + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/ParserSuite.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/ParserSuite.scala new file mode 100644 index 00000000..575a3474 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/ParserSuite.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import cats.data.NonEmptyList +import cats.syntax.all.* +import cats.parse.Parser + +trait ParserSuite { + + extension [A](self: Parser[A]) + def run(str: String): A = + self.parseAll(str) match + case Left(err) => + Console.err.println(str) + Console.err.println(err.show) + sys.error("parsing failed") + + case Right(v) => v + + def nel[A](a: A, more: A*): NonEmptyList[A] = + NonEmptyList(a, more.toList) +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala new file mode 100644 index 00000000..ec0dfe36 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala @@ -0,0 +1,120 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import cats.data.NonEmptyList as Nel +import io.renku.search.query.Comparison.{GreaterThan, LowerThan} +import io.renku.search.query.* +import munit.FunSuite + +import java.util.concurrent.atomic.AtomicInteger + +class QueryParserSpec extends FunSuite with ParserSuite { + + test("string list") { + val p = QueryParser.values + assertEquals(p.run("a,b,c"), nel("a", "b", "c")) + assertEquals(p.run("a, b, c"), nel("a", "b", "c")) + assertEquals(p.run("a"), nel("a")) + assertEquals(p.run("""a,"b",c"""), nel("a", "b", "c")) + assertEquals(p.run("""a,"x\"c",c"""), nel("a", """x"c""", "c")) + assertEquals(p.run("\"hello world\""), nel("hello world")) + } + + test("field name") { + val p = QueryParser.fieldNameFrom(Field.values.toSet) + List("projectId", "projectid").foreach { s => + assertEquals(p.run(s), Field.ProjectId) + } + Field.values.foreach { f => + assertEquals(p.run(f.name), f) + } + } + + test("field term: created") { + val p = QueryParser.fieldTerm + val pd = DateTimeRef(DateTimeParser.partialDateTime.run("2023-05")) + assertEquals( + p.run("created:2023-05"), + FieldTerm.Created(Comparison.Is, Nel.of(pd)) + ) + + assertEquals( + p.run("created<2023-05"), + FieldTerm.Created(LowerThan, Nel.of(pd)) + ) + assertEquals( + p.run("created>2023-05"), + FieldTerm.Created(GreaterThan, Nel.of(pd)) + ) + } + + test("field term") { + val p = QueryParser.fieldTerm + val data = List( + "projectId:id5" -> FieldTerm.ProjectIdIs(Nel.of("id5")), + "name:\"my project\"" -> FieldTerm.NameIs(Nel.of("my project")), + "slug:ab1,ab2" -> FieldTerm.SlugIs(Nel.of("ab1", "ab2")) + ) + data.foreach { case (in, expect) => + assertEquals(p.run(in), expect) + } + } + + test("segment") { + val p = QueryParser.segment + assertEquals( + p.run("hello"), + Query.Segment.Text("hello") + ) + assertEquals( + p.run("projectId:id5"), + Query.Segment.Field(FieldTerm.ProjectIdIs(Nel.of("id5"))) + ) + assertEquals( + p.run("foo:bar"), + Query.Segment.Text("foo:bar") + ) + } + + test("example queries") { + val p = QueryParser.query + println(p.run("projectId:1 foo name:test foo bar created>today/5d")) + } + + test("generated queries") { + val counter = new AtomicInteger(0) + QueryGenerators + .nelOfN(20, QueryGenerators.query) + .sample + .toList + .flatMap(_.toList) + .foreach { q => + val qStr = q.asString + println(s">>: ${qStr}") + val parsed = Query.parse(qStr) + if (parsed.isLeft) { + val _ = counter.incrementAndGet() + } + println(s" >> $parsed") + } + + println(s"==== Errors: ${counter.get()}") + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4b8b147d..fe966df4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -11,6 +11,7 @@ object Dependencies { val catsEffect = "3.5.3" val catsEffectMunit = "1.0.7" val catsParse = "1.0.0" + val catsScalaCheck = "0.3.2" val ciris = "3.5.0" val fs2 = "3.9.4" val http4s = "0.23.25" @@ -22,6 +23,10 @@ object Dependencies { val tapir = "1.9.9" } + val catsScalaCheck = Seq( + "io.chrisdavenport" %% "cats-scalacheck" % V.catsScalaCheck + ) + val catsParse = Seq( "org.typelevel" %% "cats-parse" % V.catsParse ) From 8e33685da05b35b0809c921dec57c0d08daff8c5 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 14 Feb 2024 19:01:33 +0100 Subject: [PATCH 3/4] WIP: query parsing, first draft done --- .../scala/io/renku/commons/Visibility.scala | 4 ++ .../scala/io/renku/search/query/Query.scala | 14 ++++- .../search/query/json/QueryJsonCodec.scala | 2 +- .../renku/search/query/parse/QueryUtil.scala | 55 +++++++++++++++++++ .../renku/search/query/QueryGenerators.scala | 38 +++++++++---- .../search/query/json/QueryJsonSpec.scala | 36 ++++-------- .../search/query/parse/QueryParserSpec.scala | 40 ++++++-------- 7 files changed, 125 insertions(+), 64 deletions(-) create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala diff --git a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala index 4db38aed..16ef2729 100644 --- a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala +++ b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala @@ -18,6 +18,8 @@ package io.renku.commons +import cats.kernel.Order + enum Visibility: case Public case Private @@ -25,5 +27,7 @@ enum Visibility: def name: String = productPrefix.toLowerCase object Visibility: + given Order[Visibility] = Order.by(_.ordinal) + def unsafeFromString(s: String): Visibility = Visibility.valueOf(s.capitalize) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index d86f4368..fa6fc963 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -25,7 +25,7 @@ import io.renku.commons.Visibility import io.renku.search.query.FieldTerm.Created import io.renku.search.query.Query.Segment import io.renku.search.query.json.QueryJsonCodec -import io.renku.search.query.parse.QueryParser +import io.renku.search.query.parse.{QueryParser, QueryUtil} final case class Query( segments: List[Query.Segment] @@ -47,13 +47,23 @@ object Query: def parse(str: String): Either[String, Query] = val trimmed = str.trim if (trimmed.isEmpty) Right(empty) - else QueryParser.query.parseAll(trimmed).leftMap(_.show) + else + QueryParser.query + .parseAll(trimmed) + .leftMap(_.show) + .map(QueryUtil.collapse) enum Segment: case Field(value: FieldTerm) case Text(value: String) object Segment: + extension (self: Segment.Text) + def ++(other: Segment.Text): Segment.Text = + if (other.value.isEmpty) self + else if (self.value.isEmpty) other + else Segment.Text(s"${self.value} ${other.value}") + def text(phrase: String): Segment = Segment.Text(phrase) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index 224d6a05..662c1866 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -45,7 +45,7 @@ import scala.collection.mutable.ListBuffer */ private[query] object QueryJsonCodec: // temporary - given Decoder[Visibility] = Decoder.forString.map(Visibility.valueOf) + given Decoder[Visibility] = Decoder.forString.map(Visibility.unsafeFromString) given Encoder[Visibility] = Encoder.forString.contramap(_.name) private[this] val freeTextField = "_text" diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala new file mode 100644 index 00000000..23f934e1 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query.parse + +import io.renku.search.query.Query +import io.renku.search.query.Query.Segment + +private[query] object QueryUtil { + + def collapse(q: Query): Query = + Query(collapseTextSegments(q.segments)) + + private def collapseTextSegments(segs: List[Segment]): List[Segment] = { + @annotation.tailrec + def loop( + in: List[Segment], + curr: Option[Segment.Text], + result: List[Segment] + ): List[Segment] = + in match + case first :: rest => + (first, curr) match + case (t1: Segment.Text, Some(tc)) => + loop(rest, Some(tc ++ t1), result) + + case (e: Segment.Text, None) => + loop(rest, Some(e), result) + + case (f: Segment.Field, Some(tc)) => + loop(rest, None, f :: tc :: result) + + case (f: Segment.Field, None) => + loop(rest, None, f :: result) + + case Nil => (curr.toList ::: result).reverse + + loop(segs, None, Nil) + } +} diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index f27c30d9..36b73ca9 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -21,6 +21,7 @@ package io.renku.search.query import cats.data.NonEmptyList import cats.syntax.all.* import io.renku.commons.Visibility +import io.renku.search.query.parse.QueryUtil import org.scalacheck.Gen import org.scalacheck.cats.implicits.* @@ -34,7 +35,7 @@ object QueryGenerators: for { h <- Gen.choose(0, 23) m <- Gen.option(Gen.choose(0, 59)) - s <- Gen.option(Gen.choose(0, 59)) + s <- if (m.isDefined) Gen.option(Gen.choose(0, 59)) else Gen.const(None) } yield PartialDateTime.Time(h, m, s) val partialDate: Gen[PartialDateTime.Date] = @@ -55,13 +56,14 @@ object QueryGenerators: val ref: Gen[PartialDateTime | RelativeDate] = Gen.oneOf(partialDateTime, relativeDate) - val period: Gen[Period] = + val periodPos: Gen[Period] = Gen.oneOf(1 to 13).map(n => Period.ofDays(n)) + val periodNeg: Gen[Period] = Gen.oneOf((-8 to -1) ++ (1 to 8)).map(n => Period.ofDays(n)) for { date <- ref - amount <- period range <- Gen.oneOf(true, false) + amount <- if (range) periodPos else periodNeg } yield DateTimeCalc(date, amount, range) } @@ -86,15 +88,28 @@ object QueryGenerators: en <- Gen.listOfN(n - 1, gen) } yield NonEmptyList(e0, en) - private val simpleString: Gen[String] = Gen.alphaNumStr - private val quotedString: Gen[String] = - Gen.alphaNumStr.map(s => s"\"$s\"") + private val alphaNumChars = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') + private val simpleWord: Gen[String] = { + val len = Gen.choose(2, 12) + len.flatMap(n => Gen.stringOfN(n, Gen.oneOf(alphaNumChars))) + } + + private val word: Gen[String] = { + val chars = alphaNumChars ++ "/{}*?()-:@…_[]^!<>=&#|~`+%\"'".toSeq + val len = Gen.choose(2, 12) + len.flatMap(n => Gen.stringOfN(n, Gen.oneOf(chars))) + } - private val valueString: Gen[String] = - Gen.oneOf(simpleString, quotedString) + private val phrase: Gen[String] = { + val w = Gen.frequency(5 -> simpleWord, 1 -> word) + Gen + .choose(1, 3) + .flatMap(n => Gen.listOfN(n, w)) + .map(_.mkString(" ")) + } private val stringValues: Gen[NonEmptyList[String]] = - Gen.choose(1, 4).flatMap(n => nelOfN(n, valueString)) + Gen.choose(1, 4).flatMap(n => nelOfN(n, phrase)) val projectIdTerm: Gen[FieldTerm] = stringValues.map(FieldTerm.ProjectIdIs(_)) @@ -111,7 +126,7 @@ object QueryGenerators: val visibilityTerm: Gen[FieldTerm] = Gen .frequency(10 -> visibility.map(NonEmptyList.one), 1 -> nelOfN(2, visibility)) - .map(FieldTerm.VisibilityIs(_)) + .map(vs => FieldTerm.VisibilityIs(vs.distinct)) private val comparison: Gen[Comparison] = Gen.oneOf(Comparison.values.toSeq) @@ -135,7 +150,7 @@ object QueryGenerators: val freeText: Gen[String] = Gen.choose(1, 5).flatMap { len => - Gen.listOfN(len, valueString).map(_.mkString(" ")) + Gen.listOfN(len, phrase).map(_.mkString(" ")) } val segment: Gen[Query.Segment] = @@ -149,3 +164,4 @@ object QueryGenerators: .choose(0, 12) .flatMap(n => Gen.listOfN(n, segment)) .map(Query.apply) + .map(QueryUtil.collapse) diff --git a/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala index 79d791f4..2e93f15c 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/json/QueryJsonSpec.scala @@ -19,33 +19,17 @@ package io.renku.search.query.json import io.bullet.borer.Json -import io.renku.search.query.{PartialDateTime, Query} -import io.renku.search.query.Query.Segment -import munit.FunSuite +import io.renku.search.query.{Query, QueryGenerators} +import munit.{FunSuite, ScalaCheckSuite} +import org.scalacheck.Prop -import java.time.Instant +class QueryJsonSpec extends ScalaCheckSuite { -class QueryJsonSpec extends FunSuite { - - test("playing") { - println(Query.empty.asString) - val q = Query( - Segment.projectIdIs("p1"), - Segment.text("foo bar"), - Segment.nameIs("ai-project-15048"), - Segment.creationDateLt(PartialDateTime.fromInstant(Instant.now())) - ) - println(q.asString) - val jsonStr = Json.encode(q).toUtf8String - println(jsonStr) - val decoded = Json.decode(jsonStr.getBytes).to[Query].value - println(decoded) - assertEquals(decoded, q) - - val q2 = Query(Segment.projectIdIs("id-2"), Segment.projectIdIs("id-3")) - val q2Json = Json.encode(q2).toUtf8String - assertEquals(q2Json, """{"projectId":"id-2","projectId":"id-3"}""") - val decodedQ2 = Json.decode(q2Json.getBytes).to[Query].value - println(decodedQ2) + property("query json encode/decode") { + Prop.forAll(QueryGenerators.query) { q => + val jsonStr = Json.encode(q).toUtf8String + val decoded = Json.decode(jsonStr.getBytes).to[Query].value + assertEquals(decoded, q) + } } } diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala index ec0dfe36..b024595b 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala @@ -19,13 +19,12 @@ package io.renku.search.query.parse import cats.data.NonEmptyList as Nel -import io.renku.search.query.Comparison.{GreaterThan, LowerThan} import io.renku.search.query.* -import munit.FunSuite - -import java.util.concurrent.atomic.AtomicInteger +import io.renku.search.query.Comparison.{GreaterThan, LowerThan} +import munit.{FunSuite, ScalaCheckSuite} +import org.scalacheck.Prop -class QueryParserSpec extends FunSuite with ParserSuite { +class QueryParserSpec extends ScalaCheckSuite with ParserSuite { test("string list") { val p = QueryParser.values @@ -93,28 +92,21 @@ class QueryParserSpec extends FunSuite with ParserSuite { ) } - test("example queries") { + test("example queries".ignore) { val p = QueryParser.query - println(p.run("projectId:1 foo name:test foo bar created>today/5d")) + println(p.run("name:\"vQgCg mpZU4cCgF3N eVZUMkH7\",JHRt visibility:private WX59P")) } - test("generated queries") { - val counter = new AtomicInteger(0) - QueryGenerators - .nelOfN(20, QueryGenerators.query) - .sample - .toList - .flatMap(_.toList) - .foreach { q => - val qStr = q.asString - println(s">>: ${qStr}") - val parsed = Query.parse(qStr) - if (parsed.isLeft) { - val _ = counter.incrementAndGet() - } - println(s" >> $parsed") + property("generated queries") { + Prop.forAll(QueryGenerators.query) { q => + val qStr = q.asString + val parsed = Query.parse(qStr).fold(sys.error, identity) + if (q != parsed) { + // this is for better error messages when things fail + println(qStr) + assertEquals(q, parsed) } - - println(s"==== Errors: ${counter.get()}") + parsed == q + } } } From 7d3a4087da2745f6c0040292a9f79514c78b6c51 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 14 Feb 2024 19:05:29 +0100 Subject: [PATCH 4/4] Adopt code for new visibility type --- .../scala/io/renku/commons/Visibility.scala | 33 ------------------- .../io/renku/search/model/projects.scala | 9 +++-- .../search/provision/SearchProvisioner.scala | 2 +- .../provision/SearchProvisionerSpec.scala | 2 +- .../io/renku/search/query/FieldTerm.scala | 2 +- .../scala/io/renku/search/query/Query.scala | 2 +- .../search/query/json/QueryJsonCodec.scala | 6 +--- .../search/query/parse/QueryParser.scala | 2 +- .../renku/search/query/QueryGenerators.scala | 2 +- 9 files changed, 14 insertions(+), 46 deletions(-) delete mode 100644 modules/commons/src/main/scala/io/renku/commons/Visibility.scala diff --git a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala b/modules/commons/src/main/scala/io/renku/commons/Visibility.scala deleted file mode 100644 index 16ef2729..00000000 --- a/modules/commons/src/main/scala/io/renku/commons/Visibility.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2024 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.commons - -import cats.kernel.Order - -enum Visibility: - case Public - case Private - - def name: String = productPrefix.toLowerCase - -object Visibility: - given Order[Visibility] = Order.by(_.ordinal) - - def unsafeFromString(s: String): Visibility = - Visibility.valueOf(s.capitalize) diff --git a/modules/commons/src/main/scala/io/renku/search/model/projects.scala b/modules/commons/src/main/scala/io/renku/search/model/projects.scala index 61cca498..0a24005f 100644 --- a/modules/commons/src/main/scala/io/renku/search/model/projects.scala +++ b/modules/commons/src/main/scala/io/renku/search/model/projects.scala @@ -18,6 +18,7 @@ package io.renku.search.model +import cats.kernel.Order import io.bullet.borer.derivation.MapBasedCodecs.* import io.bullet.borer.{Codec, Decoder, Encoder} import io.renku.search.borer.codecs.all.given @@ -70,9 +71,13 @@ object projects: given Codec[CreationDate] = Codec.of[Instant] enum Visibility derives Codec: - lazy val name: String = productPrefix + lazy val name: String = productPrefix.toLowerCase case Public, Private object Visibility: - def fromCaseInsensitive(v: String): Visibility = + given Order[Visibility] = Order.by(_.ordinal) + given Decoder[Visibility] = Decoder.forString.map(Visibility.unsafeFromString) + given Encoder[Visibility] = Encoder.forString.contramap(_.name) + + def unsafeFromString(v: String): Visibility = valueOf(v.toLowerCase.capitalize) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 2904f1ee..e13bdf82 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -112,7 +112,7 @@ private class SearchProvisionerImpl[F[_]: Async]( projects.Name(pc.name), projects.Slug(pc.slug), pc.repositories.map(projects.Repository(_)), - projects.Visibility.fromCaseInsensitive(pc.visibility.name()), + projects.Visibility.unsafeFromString(pc.visibility.name()), pc.description.map(projects.Description(_)), toUser(pc.createdBy), projects.CreationDate(pc.creationDate), diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index 9856ce36..68944db8 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -124,7 +124,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo projects.Name(created.name), projects.Slug(created.slug), created.repositories.map(projects.Repository(_)), - projects.Visibility.fromCaseInsensitive(created.visibility.name()), + projects.Visibility.unsafeFromString(created.visibility.name()), created.description.map(projects.Description(_)), toUser(created.createdBy), projects.CreationDate(created.creationDate), diff --git a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala index c27f202d..aee0cc5f 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala @@ -19,7 +19,7 @@ package io.renku.search.query import cats.data.NonEmptyList -import io.renku.commons.Visibility +import io.renku.search.model.projects.Visibility enum FieldTerm(val field: Field, val cmp: Comparison): case ProjectIdIs(values: NonEmptyList[String]) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index fa6fc963..acf3f00f 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -21,7 +21,7 @@ package io.renku.search.query import cats.data.NonEmptyList import cats.syntax.all.* import io.bullet.borer.{Decoder, Encoder} -import io.renku.commons.Visibility +import io.renku.search.model.projects.Visibility import io.renku.search.query.FieldTerm.Created import io.renku.search.query.Query.Segment import io.renku.search.query.json.QueryJsonCodec diff --git a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index 662c1866..21239fb2 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -21,7 +21,7 @@ package io.renku.search.query.json import cats.data.NonEmptyList import io.bullet.borer.compat.cats.* import io.bullet.borer.{Decoder, Encoder, Reader, Writer} -import io.renku.commons.Visibility +import io.renku.search.model.projects.Visibility import io.renku.search.query.* import io.renku.search.query.FieldTerm.* import io.renku.search.query.Query.Segment @@ -44,10 +44,6 @@ import scala.collection.mutable.ListBuffer * }}} */ private[query] object QueryJsonCodec: - // temporary - given Decoder[Visibility] = Decoder.forString.map(Visibility.unsafeFromString) - given Encoder[Visibility] = Encoder.forString.contramap(_.name) - private[this] val freeTextField = "_text" enum Name: diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala index 326c3f83..d7f54a42 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -20,7 +20,7 @@ package io.renku.search.query.parse import cats.data.NonEmptyList import cats.parse.{Parser as P, Parser0 as P0} -import io.renku.commons.Visibility +import io.renku.search.model.projects.Visibility import io.renku.search.query.* private[query] object QueryParser { diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 36b73ca9..6c016caa 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -20,7 +20,7 @@ package io.renku.search.query import cats.data.NonEmptyList import cats.syntax.all.* -import io.renku.commons.Visibility +import io.renku.search.model.projects.Visibility import io.renku.search.query.parse.QueryUtil import org.scalacheck.Gen import org.scalacheck.cats.implicits.*