Skip to content

Commit

Permalink
Improve reading url patterns from string (#153)
Browse files Browse the repository at this point in the history 
* Allow to match the rest of a path for a url pattern

A `**` can be used to match everything after without restriction. This
can be used to match remaining segments of a path, for example.

* Filter out empty url-pattern segments

Converts something like `test.com/a//b` to `test.com/a/b`
  • Loading branch information
@eikek
eikek authored Jun 12, 2024
1 parent ccdbbd9 commit 80612a7
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 53 deletions.
68 changes: 47 additions & 21 deletions modules/commons/src/main/scala/io/renku/search/common/UrlPattern.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,28 @@ final case class UrlPattern(
def matches(url: String): Boolean =
val parts = UrlPattern.splitUrl(url)
scheme.forall(s => parts.scheme.exists(s.matches)) &&
(host.isEmpty || host.length == parts.host.length) &&
host.zip(parts.host).forall { case (s, h) => s.matches(h) } &&
matchList(parts.host, host) &&
port.forall(p => parts.port.exists(p.matches)) &&
(path.isEmpty || path.length == parts.path.length) &&
path.zip(parts.path).forall { case (s, p) => s.matches(p) }
matchList(parts.path, path)

def render: String =
scheme.map(s => s"${s.render}://").getOrElse("") +
host.map(_.render).mkString(".") +
port.map(p => s":${p.render}").getOrElse("") +
(if (path.isEmpty) "" else path.map(_.render).mkString("/", "/", ""))

private def matchList(values: List[String], pattern: List[Segment]): Boolean =
pattern.isEmpty || {
pattern.indexOf(Segment.MatchAllRemainder) match
case n if n < 0 =>
values.lengthIs == pattern.length && pattern.zip(values).forall { case (s, h) =>
s.matches(h)
}
case n =>
values.sizeIs >= n &&
pattern.take(n).zip(values).forall { case (s, h) => s.matches(h) }
}

object UrlPattern:
val all: UrlPattern = UrlPattern(None, Nil, None, Nil)

Expand Down Expand Up @@ -69,44 +79,60 @@ object UrlPattern:
rest0.split('/').toList match
case hp :: rest =>
val (host, port) = readHostPort(hp)
UrlParts(scheme, host, port, rest)
UrlParts(scheme, host, port, rest.filter(_.nonEmpty))
case _ =>
val (host, port) = readHostPort(rest0)
UrlParts(scheme, host, port, Nil)
}

def fromString(str: String): UrlPattern =
if (str == "*" || str.isEmpty) UrlPattern.all
def fromString(str: String): Either[String, UrlPattern] =
if (str.isEmpty) Left("empty url pattern")
else if ("*" == str || "**" == str) Right(UrlPattern.all)
else {
val parts = splitUrl(str)
UrlPattern(
parts.scheme.map(Segment.fromString),
parts.host.map(Segment.fromString),
parts.port.map(Segment.fromString),
parts.path.map(Segment.fromString)
)
// specifiyng anything after a '**' doesn't make sense, so we detect
// it here and fail to convert to a pattern
// (no ** || last = **)
def check(segs: List[Segment]) =
val (pre, suf) = segs.span(_ != Segment.MatchAllRemainder)
if (suf.sizeIs <= 1) Right(segs)
else Left("A '**' should not be followed by other segments")

for
host <- check(parts.host.map(Segment.fromString))
path <- check(parts.path.map(Segment.fromString))
scheme = parts.scheme.map(Segment.fromString)
port = parts.port.map(Segment.fromString)
yield UrlPattern(scheme, host, port, path)
}

def unsafeFromString(str: String): UrlPattern =
fromString(str).fold(sys.error, identity)

enum Segment:
case Literal(value: String)
case Prefix(value: String)
case Suffix(value: String)
case MatchAll
case MatchAllRemainder

def matches(value: String): Boolean = this match
case Literal(v) => v.equalsIgnoreCase(value)
case Prefix(v) => value.startsWith(v)
case Suffix(v) => value.endsWith(v)
case MatchAll => true
case Literal(v) => v.equalsIgnoreCase(value)
case Prefix(v) => value.startsWith(v)
case Suffix(v) => value.endsWith(v)
case MatchAll => true
case MatchAllRemainder => true

def render: String = this match
case Literal(v) => v
case Prefix(v) => s"${v}*"
case Suffix(v) => s"*${v}"
case MatchAll => "*"
case Literal(v) => v
case Prefix(v) => s"${v}*"
case Suffix(v) => s"*${v}"
case MatchAll => "*"
case MatchAllRemainder => "**"

object Segment:
def fromString(s: String): Segment = s match
case "**" => MatchAllRemainder
case "*" => MatchAll
case x if x.startsWith("*") => Suffix(x.drop(1))
case x if x.endsWith("*") => Prefix(x.dropRight(1))
Expand Down
41 changes: 31 additions & 10 deletions modules/commons/src/test/scala/io/renku/search/common/CommonGenerators.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,43 @@ object CommonGenerators:
} yield NonEmptyList(e0, en)

def urlPatternGen: Gen[UrlPattern] =
def segmentGen(inner: Gen[String]): Gen[UrlPattern.Segment] =
val allMatchBoth =
Gen.oneOf(UrlPattern.Segment.MatchAll, UrlPattern.Segment.MatchAllRemainder)
val allMatchOnly = Gen.const(UrlPattern.Segment.MatchAll)

def segmentGen(
inner: Gen[String],
other: Gen[UrlPattern.Segment]
): Gen[UrlPattern.Segment] =
Gen.oneOf(
inner.map(s => UrlPattern.Segment.Prefix(s)),
inner.map(s => UrlPattern.Segment.Suffix(s)),
inner.map(s => UrlPattern.Segment.Literal(s)),
Gen.const(UrlPattern.Segment.MatchAll)
other
)

val schemes = segmentGen(Gen.oneOf("http", "https"))
val ports = segmentGen(Gen.oneOf("123", "8080", "8145", "487", "11"))
val hosts = segmentGen(
Gen.oneOf("test", "com", "ch", "de", "dev", "renku", "penny", "cycle")
).asListOfN(0, 5)
val paths = segmentGen(
Gen.oneOf("auth", "authenticate", "doAuth", "me", "run", "api")
).asListOfN(0, 5)
def appendMatchRemainder(
g: Gen[List[UrlPattern.Segment]]
): Gen[List[UrlPattern.Segment]] =
for
segs <- g
rem <- Gen.option(Gen.const(UrlPattern.Segment.MatchAllRemainder))
yield segs ++ rem.toList

val schemes = segmentGen(Gen.oneOf("http", "https"), allMatchBoth)
val ports = segmentGen(Gen.oneOf("123", "8080", "8145", "487", "11"), allMatchBoth)
val hosts = appendMatchRemainder(
segmentGen(
Gen.oneOf("test", "com", "ch", "de", "dev", "renku", "penny", "cycle"),
allMatchOnly
).asListOfN(0, 4)
)
val paths = appendMatchRemainder(
segmentGen(
Gen.oneOf("auth", "authenticate", "doAuth", "me", "run", "api"),
allMatchOnly
).asListOfN(0, 4)
)
for
scheme <- schemes.asOption
host <- hosts
Expand Down
84 changes: 66 additions & 18 deletions modules/commons/src/test/scala/io/renku/search/common/UrlPatternSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ import munit.ScalaCheckSuite
import org.scalacheck.Prop

class UrlPatternSpec extends ScalaCheckSuite:
def urlPattern(str: String) = UrlPattern.unsafeFromString(str)

test("read parts"):
test("read parts") {
assertEquals(UrlPattern.splitUrl(""), UrlParts(None, Nil, None, Nil))
assertEquals(
UrlPattern.splitUrl("test.com"),
Expand All @@ -51,20 +52,24 @@ class UrlPatternSpec extends ScalaCheckSuite:
UrlPattern.splitUrl("/auth/exec"),
UrlParts(None, Nil, None, List("auth", "exec"))
)
assertEquals(
UrlPattern.splitUrl("https://test.com:123/auth//**"),
UrlParts(Some("https"), List("test", "com"), Some("123"), List("auth", "**"))
)
}

test("fromString"):
test("fromString successful") {
assertEquals(
UrlPattern.fromString("http://"),
urlPattern("http://"),
UrlPattern.all.copy(scheme = Some(Segment.Literal("http")))
)
assertEquals(
UrlPattern.fromString("http"),
urlPattern("http"),
UrlPattern.all.copy(host = List(Segment.Literal("http")))
)
assertEquals(UrlPattern.fromString("*"), UrlPattern.all)
assertEquals(UrlPattern.fromString(""), UrlPattern.all)
assertEquals(urlPattern("*"), UrlPattern.all)
assertEquals(
UrlPattern.fromString("*.*"),
urlPattern("*.*"),
UrlPattern(
None,
List(Segment.MatchAll, Segment.MatchAll),
Expand All @@ -73,7 +78,7 @@ class UrlPatternSpec extends ScalaCheckSuite:
)
)
assertEquals(
UrlPattern.fromString("*.test.com"),
urlPattern("*.test.com"),
UrlPattern(
None,
List(Segment.MatchAll, Segment.Literal("test"), Segment.Literal("com")),
Expand All @@ -82,7 +87,7 @@ class UrlPatternSpec extends ScalaCheckSuite:
)
)
assertEquals(
UrlPattern.fromString("*test.com"),
urlPattern("*test.com"),
UrlPattern(
None,
List(Segment.Suffix("test"), Segment.Literal("com")),
Expand All @@ -91,7 +96,7 @@ class UrlPatternSpec extends ScalaCheckSuite:
)
)
assertEquals(
UrlPattern.fromString("*test.com/auth*"),
urlPattern("*test.com/auth*"),
UrlPattern(
None,
List(Segment.Suffix("test"), Segment.Literal("com")),
Expand All @@ -100,18 +105,43 @@ class UrlPatternSpec extends ScalaCheckSuite:
)
)
assertEquals(
UrlPattern.fromString("https://test.com:15*/auth/sign"),
urlPattern("https://test.com:15*/auth/sign"),
UrlPattern(
Some(Segment.Literal("https")),
List(Segment.Literal("test"), Segment.Literal("com")),
Some(Segment.Prefix("15")),
List(Segment.Literal("auth"), Segment.Literal("sign"))
)
)
assertEquals(
urlPattern("https://test.com/**"),
UrlPattern(
Some(Segment.Literal("https")),
List(Segment.Literal("test"), Segment.Literal("com")),
None,
List(Segment.MatchAllRemainder)
)
)
assertEquals(
urlPattern("https://test.com/abc/**"),
UrlPattern(
Some(Segment.Literal("https")),
List(Segment.Literal("test"), Segment.Literal("com")),
None,
List(Segment.Literal("abc"), Segment.MatchAllRemainder)
)
)
}

test("fromString fail") {
assert(UrlPattern.fromString("").isLeft)
assert(UrlPattern.fromString("test.com/a/**/b").isLeft)
assert(UrlPattern.fromString("**.com/a/b").isLeft)
}

property("read valid url pattern") {
Prop.forAll(CommonGenerators.urlPatternGen) { pattern =>
val parsed = UrlPattern.fromString(pattern.render)
val parsed = urlPattern(pattern.render)
val result = parsed == pattern
if (!result) {
println(s"Given: $pattern Parsed: ${parsed} Rendered: ${pattern.render}")
Expand All @@ -132,17 +162,31 @@ class UrlPatternSpec extends ScalaCheckSuite:

test("matches successful"):
List(
UrlPattern.fromString("*.test.com") -> List(
urlPattern("*.test.com") -> List(
"dev.test.com",
"http://sub.test.com/ab/cd"
),
UrlPattern.fromString("/auth/renku") -> List(
urlPattern("/auth/renku") -> List(
"dev.test.com/auth/renku",
"http://sub.test.com/auth/renku"
),
UrlPattern.fromString("*.test.com/auth/renku") -> List(
urlPattern("*.test.com/auth/renku") -> List(
"http://dev.test.com/auth/renku",
"sub1.test.com/auth/renku"
),
urlPattern("https://renku.com/auth/**") -> List(
"https://renku.com/auth/realms/Renku"
),
urlPattern("test.com/**") -> List(
"http://test.com/auth/a/b",
"https://test.com/auth",
"https://test.com",
"https://test.com/"
),
urlPattern("test.com/a/c/**") -> List(
"http://test.com/a/c",
"http://test.com/a/c/b",
"http://test.com/a/c/b/1/2"
)
).foreach { case (pattern, values) =>
values.foreach(v =>
Expand All @@ -155,17 +199,21 @@ class UrlPatternSpec extends ScalaCheckSuite:

test("matches not successful"):
List(
UrlPattern.fromString("*.test.com") -> List(
urlPattern("*.test.com") -> List(
"fest.com",
"http://sub.fest.com/ab/cd"
),
UrlPattern.fromString("/auth/renku") -> List(
urlPattern("/auth/renku") -> List(
"fest.com/tauth/renku",
"http://sub.test.com/auth/renkuu"
),
UrlPattern.fromString("*.test.com/auth/renku") -> List(
urlPattern("*.test.com/auth/renku") -> List(
"http://dev.test.com/auth",
"sub1.sub2.test.com/auth/renku"
),
urlPattern("test.com/a/c/**") -> List(
"http://test.com/a/b/c",
"http://test.com/a"
)
).foreach { case (pattern, values) =>
values.foreach(v =>
Expand Down
10 changes: 8 additions & 2 deletions modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package io.renku.search.config

import cats.Show
import cats.syntax.all.*
import ciris.{ConfigDecoder, ConfigError}
import com.comcast.ip4s.{Ipv4Address, Port}
Expand All @@ -28,6 +29,11 @@ import scala.concurrent.duration.{Duration, FiniteDuration}
import io.renku.search.common.UrlPattern

trait ConfigDecoders:
extension [A, B](self: ConfigDecoder[A, B])
def emap[C](typeName: String)(f: B => Either[String, C])(using Show[B]) =
self.mapEither((key, b) =>
f(b).left.map(err => ConfigError.decode(typeName, key, b))
)

given ConfigDecoder[String, Uri] =
ConfigDecoder[String].mapEither { (_, s) =>
Expand Down Expand Up @@ -63,6 +69,6 @@ trait ConfigDecoders:
.mapOption(Port.getClass.getSimpleName)(Port.fromString)

given ConfigDecoder[String, List[UrlPattern]] =
ConfigDecoder[String].map { str =>
str.split(',').toList.map(UrlPattern.fromString)
ConfigDecoder[String].emap("UrlPattern") { str =>
str.split(',').toList.traverse(UrlPattern.fromString)
}
4 changes: 2 additions & 2 deletions modules/openid-keycloak/src/test/scala/io/renku/openid/keycloak/DefaultJwtVerifySpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class DefaultJwtVerifySpec

val issuer: Uri = uri"https://ci-renku-3622.dev.renku.ch/auth/realms/Renku"
val jwtConfig = JwtVerifyConfig.default.copy(allowedIssuerUrls =
List(UrlPattern.fromString("*.*.renku.ch"))
List(UrlPattern.unsafeFromString("*.*.renku.ch"))
)

extension [B](self: Either[JwtError, B])
Expand Down Expand Up @@ -152,7 +152,7 @@ class DefaultJwtVerifySpec

test("stop on invalid issuer"):
val testClient = Client.fromHttpApp(HttpRoutes.empty[IO].orNotFound)
val allowedIssuers = List(UrlPattern.fromString("*.myserver.com"))
val allowedIssuers = List(UrlPattern.unsafeFromString("*.myserver.com"))
for
verifyer <- DefaultJwtVerify(
testClient,
Expand Down

0 comments on commit 80612a7

Please sign in to comment.