diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20873501..6a424a85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,26 +16,15 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - java: [ 17 ] steps: - uses: actions/checkout@v4.1.1 with: - fetch-depth: 100 - - uses: olafurpg/setup-scala@v14 - with: - java-version: ${{ matrix.java }} - # - name: Coursier cache - # uses: coursier/cache-action@v6 + fetch-depth: 0 + - uses: cachix/install-nix-action@v27 - name: sbt ci ${{ github.ref }} - run: sbt -mem 2048 ci - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.RENKU_DOCKER_USERNAME }} - password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - name: sbt docker:publishLocal - run: sbt -mem 2048 search-provision/Docker/publishLocal search-api/Docker/publishLocal + env: + README_BASE_REF: origin/${{ github.base_ref }} + run: nix develop .#ci --command sbt ci ci: runs-on: ubuntu-latest needs: [ci-matrix] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d11cab2..96472558 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,5 @@ name: Release on: - push: - branches: [ main ] release: types: [ published ] diff --git a/README.md b/README.md new file mode 100644 index 00000000..6e3a34d7 --- /dev/null +++ b/README.md @@ -0,0 +1,267 @@ + +# Renku Search + + +This provides the renku search services for efficientlyf searching across +entities in the Renku platform. + +The engine backing the search functionality is [SOLR](https://solr.apache.org) +(Lucene). The index is created from data pulled from a Redis stream. + +There are two services provided: [search-api](#search-api) and +[search-provision](#search-provision). + +There is a detailed [development documentation](development.md) for getting +started. + +## Search Provision + +Responsible for maintaining the index. This service is pulling elements out of +the Redis stream, transforming it into solr documents and the updates the index. +It also creates the SOLR schema and provides endpoints to trigger re-indexing. + +This service is internal only and can be used by other service to publish data +that should be search- and discoverable. + +The data in the index is received as a Redis message pulled from a Redis stream. + +### Messages + +Messages in the stream must conform to the definitions in +[renku-schema](https://github.com/SwissDataScienceCenter/renku-schema) and are +sent as binary [Avro](https://avro.apache.org/) messages. + +A redis message is expected to contain two keys: + +- `headers` +- `payload` + +Where the header denotes properties that control how a payload is processed. The +important properties are `type`, `dataContentType` and `schemaVersion`. + +**type** specifies what kind of payload is transported and how it can be +decoded. There are currently these message types, each denoting a specific +payload structure: + +- `project.created` +- `project.updated` +- `project.removed` +- `projectAuth.added` +- `projectAuth.updated` +- `projectAuth.removed` +- `user.added` +- `user.updated` +- `user.removed` +- `group.added` +- `group.updated` +- `group.removed` +- `memberGroup.added` +- `memberGroup.updated` +- `memberGroup.removed` +- `reprovisioning.started` +- `reprovisioning.finished` + +If a header contains a value different from that list, the message cannot be +processed. + +**dataContentType** specifies the transport encoding, where avro supports + +- `application/avro+binary` +- `application/avro+json` + +**schemaVersion** specifies which version of `renku-schema` messages is sent. +Search supports + +- `V1` +- `V2` + +The `V` can be omitted in the payload. + +### Endpoints + +There are few endpoints exposed for internal use only. + +- `/reindex` +- `/ping` +- `/version` + +Doing a re-index works by dropping the SOLR index completely and then re-reading +the Redis stream. The `reindex` endpoint requires POST request with a JSON +payload. It can optionally specify a redis message-id from where to start +reading. If it is omitted, it will start from the last known message that +initiated the index. + +Example: + +Re-Index from last known start: +``` +POST /reindex +Content-Type: application/json + +{ + +} +``` + +Re-index by speciying a message id to start from: +``` +POST /reindex +Content-Type: application/json + +{ + "messageId": "22154-0" +} +``` + + +### Configuration + +The service is configured via environment variables. Each variable is prefixed +with `RS_` (for "renku search"). + +``` +RS_CLIENT_ID=search-provisioner +RS_HTTP_SHUTDOWN_TIMEOUT=30s +RS_LOG_LEVEL=2 +RS_METRICS_UPDATE_INTERVAL=15 seconds +RS_PROVISION_HTTP_SERVER_BIND_ADDRESS=0.0.0.0 +RS_PROVISION_HTTP_SERVER_PORT=8081 +RS_REDIS_CONNECTION_REFRESH_INTERVAL=30 minutes +RS_REDIS_DB= +RS_REDIS_HOST=localhost +RS_REDIS_MASTER_SET= +RS_REDIS_PASSWORD= +RS_REDIS_PORT=6379 +RS_REDIS_QUEUE_DATASERVICE_ALLEVENTS= +RS_REDIS_QUEUE_GROUPMEMBER_ADDED= +RS_REDIS_QUEUE_GROUPMEMBER_REMOVED= +RS_REDIS_QUEUE_GROUPMEMBER_UPDATED= +RS_REDIS_QUEUE_GROUP_ADDED= +RS_REDIS_QUEUE_GROUP_REMOVED= +RS_REDIS_QUEUE_GROUP_UPDATED= +RS_REDIS_QUEUE_PROJECTAUTH_ADDED= +RS_REDIS_QUEUE_PROJECTAUTH_REMOVED= +RS_REDIS_QUEUE_PROJECTAUTH_UPDATED= +RS_REDIS_QUEUE_PROJECT_CREATED= +RS_REDIS_QUEUE_PROJECT_REMOVED= +RS_REDIS_QUEUE_PROJECT_UPDATED= +RS_REDIS_QUEUE_USER_ADDED= +RS_REDIS_QUEUE_USER_REMOVED= +RS_REDIS_QUEUE_USER_UPDATED= +RS_REDIS_SENTINEL= +RS_RETRY_ON_ERROR_DELAY=10 seconds +RS_SOLR_CORE=search-core-test +RS_SOLR_LOG_MESSAGE_BODIES=false +RS_SOLR_PASS= +RS_SOLR_URL=http://localhost:8983 +RS_SOLR_USER=admin +``` + + +## Search Api + +Provides http endpoints for searching the index. There is a [query +dsl](/docs/query-manual.md) for more convenient searching for renku entities. +Additionally, there is an openapi documentation generated and a version +endpoint. + +``` +GET /api/search/query?q= +GET /api/search/version +GET /api/search/spec.json +``` + +Here is an example of the result structure. For more details, the openapi doc +should be consulted. + +``` json +{ + "items": [ + { + "type": "Project", + "id": "01HRA7AZ2Q234CDQWGA052F8MK", + "name": "renku", + "slug": "renku", + "namespace": { + "type": "Group", + "id": "2CAF4C73F50D4514A041C9EDDB025A36", + "name": "SDSC", + "namespace": "SDSC", + "description": "SDSC group", + "score": 1.1 + }, + "repositories": [ + "https: //github.com/renku" + ], + "visibility": "public", + "description": "Renku project", + "createdBy": { + "type": "User", + "id": "1CAF4C73F50D4514A041C9EDDB025A36", + "namespace": "renku/renku", + "firstName": "Albert", + "lastName": "Einstein", + "score": 2.1 + }, + "creationDate": "2024-09-26T13: 42: 57.213115111Z", + "keywords": [ + "data", + "science" + ], + "score": 1.0 + }, + { + "type": "User", + "id": "1CAF4C73F50D4514A041C9EDDB025A36", + "namespace": "renku/renku", + "firstName": "Albert", + "lastName": "Einstein", + "score": 2.1 + }, + { + "type": "Group", + "id": "2CAF4C73F50D4514A041C9EDDB025A36", + "name": "SDSC", + "namespace": "SDSC", + "description": "SDSC group", + "score": 1.1 + } + ], + "facets": { + "entityType": { + "Project": 1, + "Group": 1, + "User": 1 + } + }, + "pagingInfo": { + "page": { + "limit": 25, + "offset": 0 + }, + "totalResult": 3, + "totalPages": 1 + } +} +``` + +### Configuration + +The service is configured via environment variables. Each variable is prefixed +with `RS_` (for "renku search"). + +``` +RS_HTTP_SHUTDOWN_TIMEOUT=30s +RS_JWT_ALLOWED_ISSUER_URL_PATTERNS= +RS_JWT_ENABLE_SIGNATURE_CHECK=true +RS_JWT_KEYCLOAK_REQUEST_DELAY=1 minute +RS_JWT_OPENID_CONFIG_PATH=.well-known/openid-configuration +RS_LOG_LEVEL=2 +RS_SEARCH_HTTP_SERVER_BIND_ADDRESS=0.0.0.0 +RS_SEARCH_HTTP_SERVER_PORT=8080 +RS_SOLR_CORE=search-core-test +RS_SOLR_LOG_MESSAGE_BODIES=false +RS_SOLR_PASS= +RS_SOLR_URL=http://localhost:8983 +RS_SOLR_USER=admin +``` diff --git a/build.sbt b/build.sbt index c357c76c..3e899273 100644 --- a/build.sbt +++ b/build.sbt @@ -28,7 +28,10 @@ releaseVersionBump := sbtrelease.Version.Bump.Minor releaseIgnoreUntrackedFiles := true releaseTagName := (ThisBuild / version).value -addCommandAlias("ci", "; lint; compile; Test/compile; dbTests; publishLocal") +addCommandAlias( + "ci", + "readme/readmeCheckModification; lint; compile; Test/compile; dbTests; readme/readmeUpdate; publishLocal; search-provision/Docker/publishLocal; search-api/Docker/publishLocal" +) addCommandAlias( "lint", "; scalafmtSbtCheck; scalafmtCheckAll; scalafixAll --check" @@ -441,6 +444,33 @@ lazy val searchCli = project configValues % "compile->compile;test->test" ) +lazy val readme = project + .in(file("modules/readme")) + .enablePlugins(ReadmePlugin) + .settings(commonSettings) + .settings( + name := "search-readme", + scalacOptions := + Seq( + "-feature", + "-deprecation", + "-unchecked", + "-encoding", + "UTF-8", + "-language:higherKinds", + "-Xkind-projector:underscores" + ), + readmeAdditionalFiles := Map( + // copy query manual into source tree for easier discovery on github + (searchQueryDocs / Docs / outputDirectory).value / "manual.md" -> "query-manual.md" + ) + ) + .dependsOn( + renkuRedisClient % "compile->compile;compile->test", + searchProvision, + searchApi + ) + lazy val commonSettings = Seq( organization := "io.renku", publish / skip := true, diff --git a/development.md b/development.md new file mode 100644 index 00000000..456d0e03 --- /dev/null +++ b/development.md @@ -0,0 +1,279 @@ + +# Development Docs + +This is a [Scala 3](https://scala-lang.org) project and it uses a pure +functional style building on top of the [typelevel](https://typelevel.org) +ecosystem. + + +## Dev Environment + +A dev environment contains build tools and additional utilities convenient for +developing. These environments can be easily created using +[nix](https://nixos.org/download). Please install nix first as a prerequisite. + +There can be many development environments, each is defined in the `devShells` +property in `flake.nix`. There is one called `ci` which is used to run the `sbt +ci` task as a github action. It defines everything (and nothing else) that is +required for the ci task. + +You can enter such an environment with `nix develop .#`. To drop +into the environment used by the CI, run: + +``` +$ nix develop .#ci +``` + +Once that returns, sbt is available and you can run `sbt` to compile the +project. + +The nix setup uses [devshell-tools](https://github.com/eikek/devshell-tools) +providing the vm and container definitions. Services are defined in +`./nix/services.nix`. The `nix/` folder also contains scripts that are made +available in the dev shells. + +### Multiple dev shells + +The `flake.nix` file defines other dev shells that target specific setups for +running the services locally on your machine. + +To run locally, search-api requires a SOLR to connect to and search-provision +additonally needs Redis. These two external dependencies are provided either in +a VM or a container. The container here is based on `systemd-nspawn` and meant +to be used when developing on NixOS (but might work on other systemd linuxes). +The VM can be created on other systems. As an example, here the vm version is +shown. + +Drop into the corresponding dev shell: + +``` +$ nix develop .#vm +``` + +Now sbt and other utilities are available, as well as scripts to create and +start a VM with the required services. Then it also defines environment +variables to configure the search services to connect to this vm. To create and +start the VM: + +``` +$ vm-run +``` + +Do `vm-` to find other scripts related to managing the VM. The `vm-run` +command starts the VM in headless mode in your terminal. So to continue, open +another terminal for running the application. + +If you have other needs, you can always create another dev shell in `flake.nix` +and tweak it to your likings. It won't affect other developers and ci, since +they use separate definitions. + + +### Most convenience using dirnev + +The `nix develop` command drops you in a bash shell, which might not be your +most favorite shell. It is recommended to use [direnv](https://direnv.net/) for +a much easier and convenient way to get into a development shell. + +The project contains a `.envrc-template` file. Copy this to `.envrc` and change +its contents to name a dev environment you would like to use. Now run `direnv +allow` in the source root to allow direnv to load this shell from the +`flake.nix` file whenever you enter this directory. + +With this setup, when you `cd` into the project directory, `direnv` will load +the development shell. Direnv is available for many shells and there are also +many integrations with editors. + +## Build + +For building from source, [sbt](https://scala-sbt.org) > 1.0 and JVM >=17 is +required. When sbt is started it looks into `project/build.properties` to +download the correct version of itself. Obviously, when using a dev shell as +described above, sbt is already there. + +Then, from within the sbt shell, run + +- `compile` to compile all main sources +- `Test/compile` to compile main and test sources + +For creating a package, there are these commands: + +- zip files: + - `search-provision/Universal/packageBin` to create a zip containing + the search-provision service + - `search-api/Universal/packageBin` to create a zip containing the + search-api service +- Docker + - `search-provision/Docker/publish` to create a docker image for the + search-provision service + - `search-api/Docker/publish` to create a docker image for the + search-api service + + +## Run from Sources + +To run the two services directly from the source tree: + +``` +sbt:renku-search> reStart +``` + +This will start the two services, it may fail if the +[environment](#dev-environment) is not setup correctly. The search services +require SOLR and Redis (search-provision only) as an external dependency to +connect to. + +The easiest way to get there, is using the `vm` dev shell. + +1. Open two terminal sessions, enter the dev shell either via direnv or `nix + develop .#vm` +2. In one terminal, start the vm with `vm-run` + - this creates and then runs the VM in headless mode in your terminal + - you can login with `root:root` or use `vm-ssh` from the other terminal + session +3. In the other terminal, run `sbt` and inside the sbt shell `reStart` + +Now the search services are running since the dev dev shell also defines the +correct environment variables to have the services connect to SOLR and Redis +running inside the VM. + +As a quick test, go to which should show the openapi +docs for the search service. This is served from inside the VM while the openapi +specification is taken from the locally running search service +. + +SOLR can be reached at and Redis is port-forwarded from +`localhost:16973`. Look into the `flake.nix` file for more details. + +## Use the CLI for testing + +The module `search-cli` implements a simple cli tool that can be used to try +things out. For example, it can send redis messages into the stream that will +then be read by the provisioning service and finally populate the SOLR index. + +Inside the sbt shell, you can run the cli for example to create a new user: + +``` +sbt:renku-search> search-cli/run user add --id user1 --namespace user1 --first-name John --last-name Doe +``` + +## Avro Schema Models + +The messages in the Redis stream must conform to the specification in the +[renku-schema](https://github.com/SwissDataScienceCenter/renku-schema) +repository. The module `events` defines the events data types and uses generated +code from these schema files. The code is generated as part of the `compile` +build step. + +The relevant parts for it are in the `AvroCodeGen` and `AvroSchemaDownload` +build plugins (inside `./project`). The `AvroSchemaDownload` simply clones the +`renku-schema` repository while `AvroCodeGen` only wraps around the +[sbt-avrohugger](https://github.com/julianpeeters/sbt-avrohugger) plugin. + +## Commits and PRs + +Commits should ideally address one semantically closed unit of work. They should +have a meaningful title and ideally provide a good description about the +intention and motives for the change as well as perhaps certain special +mentions. + +Multiple nicely crafted commits can go into one pull request, in which case it +is merged with a merge commit. + +It is also fine to create many "fixup" commits in a pull request and +squash-merge it. + +For organizing changes and creating the release notes, the pull requests are +used. So the PR title should be written in a nice way, preferably without +spelling errors :-). Each pull request should be labelled with exactly one of +these labels: + +- `feature` or `enhancement` +- `fix` or `bug` +- `chore` +- `documentation` +- `dependencies` + +More other labels are fine, obviously. For the exact labels affecting the releas +notes, look into `.github/release-drafter.yml`. + +These labels are then used to draft release notes on GitHub. If a PR should not +show up there, label it with `skip-changelog`. This can be desired if a feature +PR gets fixes after it has been merged but before it has been released. It then +doesn't make sense to mention everything found during the initial build of that +feature. + +## Tests + +Some tests require solr and redis to be available. The test will by default +start a docker container for each external service, _unless_ environment +variables `NO_SOLR` and `NO_REDIS`, respectively, are present. + +If you use a dev shell, these variables are defined so that the tests use either +the container defined in the dev shell or the VM. + +In order to reduce run time of the tests, the external services are started +before any test is run and stopped after all tests are finished. The tests must +take care to operate on an isolated part, like creating random solr cores. + +If you want to run the tests with the solr and redis setup: + +``` +sbt:renku-search> dbTests +``` + +You can run tests without this setup: +``` +sbt:renku-search> test +``` + + +## CI + +GitHub actions are used to check pull requests. The ci is setup to delegate +everything to sbt. This way you can always run the exact same checks on your +local machine. + +The ci actions use the dev shell `ci` and then run `sbt ci` command. The sbt +`ci` command is an alias that simply defines a list of other sbt tasks to run. + +This runs the ci action locally: + +``` +$ nix develop .#ci --command sbt ci +``` + +If you run the above, make sure *not* to be in a development shell. + +## Making a Release + +A release is created on GitHub: +https://github.com/SwissDataScienceCenter/renku-search/releases + +Click on `Edit` of the latest draft release that has been prepared by the +release-drafter github action. The release notes should already look pretty and +in almost all cases don't need to be tweaked. + +Choose a new tag **and prefix it with a `v`**, like in `v0.6.0`. Then hit +*Publish release* and wait… + + +## Documentation + +For changing the `README.md` file, do any modification to `docs/readme.md` and +then run the sbt task `readme/readmeUpdate` (this is also part of the ci chain). +This task compiles and runs the scala code snippets in that file. + +The manual about the query dsl is in a separate module to be easier consumed by +the search api module. Changes here must go to +`module/search-query-docs/docs/manual.md`. + +Markdown files should also be nicely readable in plain text (the original +intention of the markdown format). Please use a proper setup in your editor to +wrap lines at 70-90 characters. Inidcate the desired line length in the file +header. + + +### ADRs + +The ADR section is for logging decisions from the team that affect this project +in some way and the consequences to design decisions here. diff --git a/docs/query-manual.md b/docs/query-manual.md new file mode 100644 index 00000000..79f1eaa8 --- /dev/null +++ b/docs/query-manual.md @@ -0,0 +1,183 @@ + +## Search Query + +The search accepts queries as a query string. A query may contain +specific and unspecific search terms. + +### Query String + +A query is a sequence of words. All words that are not recognized as +specific search terms are used for searching in various entity +properties, such as `name` or `description`. Specific search terms are +matched exactly against a certain field. Terms are separated by +whitespace. + +Example: +``` +numpy flight visibility:public +``` + +Searches for entities containing `numpy` _and_ `flight` that are +public. + +The term order is usually not relevant, it may influence the score of +a result, though. + +If a value for a specific field contains whitespace, quotes or a comma +it must be enclosed in quotes. Additionally, multiple values can be +provided for each field by using a comma separated list. The values +are treated as alternatives, so any such value would yield a result. + +Example: +``` +numpy flight visibility:public,private +``` + +Searches for entities containing `numpy` _and_ `flight` that are +_either_ `public` _or_ `private`. + +### Fields + +The following fields are available: + +- `id` +- `name` +- `slug` +- `visibility` +- `created` +- `createdBy` +- `type` +- `role` +- `keyword` +- `namespace` + +Each field allows to specify one or more values, separated by comma. +The value must be separated by a `:`. For date fields, additional `<` +and `>` is supported. + +### EntityTypes + +The field `type` allows to search for specific entity types. If it is +missing, all entity types are included in the result. Entity types are: + +- `Project` +- `User` +- `Group` + +Example: + `type:Project` + +### Roles + +The field `role` allows to search for projects the current user has +the given role. Other entities are excluded from the results. + +- `owner` +- `editor` +- `viewer` +- `member` + +### Visibility + +The `visibility` field can be used to restrict to entities with a +certain visibility. Users have a default visibility of `public`. +Possbile values are: + +- `public` +- `private` + + + +### Dates + +Date fields, like + +- `created` + +accept date strings which can be specified in various ways. There are + +- relative dates: `today` +- partial timestamps: `2023-05`, `2023-11-12T10` +- calculations based on the above: `today-5d`, `2023-10-15/10d` + + +#### Relative dates + +There are the following keywords for relative dates: + +- `today` +- `yesterday` + +#### Partial Timestamps + +Timestamps must be in ISO8601 form and are UTC based and allow to +specify time up to seconds. The full form is + +``` +yyyy-mm-ddTHH:MM:ssZ +``` + +Any part starting from right can be omitted. When querying, it will be +filled with either the maximum or minimum possible value depending on +the side of comparison. When the date is an upper bound, the missing +parts will be set to their minimum values. Conversely, when used as a +lower bound then the parts are set to its maximum value. + +Example: +- `created>2023-03` will turn into `created>2023-03-31T23:59:59` +- `created<2023-03` will turn into `created<2023-03-01T00:00:00` + +#### Date calculations + +At last, a date can be specified by adding or subtracting days from a +reference date. The reference date must be given either as a relative +date or partial timestamp. Then a `+`, `-` or `/` follows with the +amount of days. + +The `/` character allows to add and substract the days from the +reference date, making the reference date the middle. + +Example: +- `created>today-14d` things created from 14 days ago +- `created<2023-05/14d` things created from last two weeks of April + and first two weeks of May + +#### Date Comparison + +Comparing dates with `>` and `<` is done as expected. More interesting +is to specify more than one date and the use of the `:` comparison. + +The `:` can be used to specify ranges more succinctly. For a full +timestamp, it means /equals/. With partial timestamps it searches +within the minimum and maximum possible date for that partial +timestamp. + +Since multiple values are combined using `OR`, it is possible to +search in multiple ranges. + +Example: +``` +created:2023-03,2023-06 +``` + +The above means to match entities created in March 2023 or June 2023. + +## Sorting + +The query allows to define terms for sorting. Sorting is limited to +specific fields, which are: + +- `name` +- `created` +- `score` + +Sorting by a field is defined by writing the field name, followed by a +dash and the sort direction. Multiple such definitions can be +specified, using a comma separated list. Alternatively, multiple +`sort:…` terms will be combined into a single one in the order they +appear. + +Example: +`sort:score-desc,created-asc` +is equivalent to +`sort:score-desc sort:created-asc` diff --git a/docs/readme.md b/docs/readme.md new file mode 100644 index 00000000..feb7ea7e --- /dev/null +++ b/docs/readme.md @@ -0,0 +1,183 @@ + +# Renku Search + +```scala mdoc:invisible +import io.renku.search.provision.Routes +import io.renku.search.http.routes.OperationRoutes +import io.renku.redis.client.MessageBodyKeys +import io.renku.search.events.* +import io.renku.search.config.ConfigValues +import io.renku.search.provision.SearchProvisionConfig +import io.renku.search.model.EntityType +import io.renku.search.api.{data, tapir, SearchApiConfig, Microservice as SearchApiMS} +import io.renku.search.readme.* +import cats.effect.* +import cats.effect.unsafe.implicits.* +``` + +This provides the renku search services for efficientlyf searching across +entities in the Renku platform. + +The engine backing the search functionality is [SOLR](https://solr.apache.org) +(Lucene). The index is created from data pulled from a Redis stream. + +There are two services provided: [search-api](#search-api) and +[search-provision](#search-provision). + +There is a detailed [development documentation](development.md) for getting +started. + +## Search Provision + +Responsible for maintaining the index. This service is pulling elements out of +the Redis stream, transforming it into solr documents and the updates the index. +It also creates the SOLR schema and provides endpoints to trigger re-indexing. + +This service is internal only and can be used by other service to publish data +that should be search- and discoverable. + +The data in the index is received as a Redis message pulled from a Redis stream. + +### Messages + +Messages in the stream must conform to the definitions in +[renku-schema](https://github.com/SwissDataScienceCenter/renku-schema) and are +sent as binary [Avro](https://avro.apache.org/) messages. + +A redis message is expected to contain two keys: + +```scala mdoc:passthrough +BulletPoints(MessageBodyKeys.values.toSeq.map(_.name.backticks)) +``` + +Where the header denotes properties that control how a payload is processed. The +important properties are `type`, `dataContentType` and `schemaVersion`. + +**type** specifies what kind of payload is transported and how it can be +decoded. There are currently these message types, each denoting a specific +payload structure: + +```scala mdoc:passthrough +BulletPoints(MsgType.values.toSeq.map(_.name.backticks)) +``` + +If a header contains a value different from that list, the message cannot be +processed. + +**dataContentType** specifies the transport encoding, where avro supports + +```scala mdoc:passthrough +BulletPoints(DataContentType.values.toSeq.map(_.mimeType.backticks)) +``` + +**schemaVersion** specifies which version of `renku-schema` messages is sent. +Search supports + +```scala mdoc:passthrough +BulletPoints(SchemaVersion.values.toSeq.map(_.name.backticks)) +``` + +The `V` can be omitted in the payload. + +### Endpoints + +There are few endpoints exposed for internal use only. + +```scala mdoc:passthrough +BulletPoints(Routes.Paths.values.toSeq.map(e => s"/${e.name}".backticks)) +BulletPoints(OperationRoutes.Paths.values.toSeq.map(e => s"/${e.name}".backticks)) +``` + +Doing a re-index works by dropping the SOLR index completely and then re-reading +the Redis stream. The `reindex` endpoint requires POST request with a JSON +payload. It can optionally specify a redis message-id from where to start +reading. If it is omitted, it will start from the last known message that +initiated the index. + +Example: + +Re-Index from last known start: +```scala mdoc:passthrough +CodeBlock.plainLines( + s"POST /${Routes.Paths.Reindex.name}", + "Content-Type: application/json", + "", + JsonPrinter.str(Routes.ReIndexMessage(None)) +) +``` + +Re-index by speciying a message id to start from: +```scala mdoc:passthrough +CodeBlock.plainLines( + s"POST /${Routes.Paths.Reindex.name}", + "Content-Type: application/json", + "", + JsonPrinter.str(Routes.ReIndexMessage(Some(MessageId("22154-0")))) +) +``` + + +### Configuration + +The service is configured via environment variables. Each variable is prefixed +with `RS_` (for "renku search"). + +```scala mdoc:passthrough +CodeBlock.plain { + SearchProvisionConfig.configValues + .getAll.toList.sortBy(_._1).map { case (k, v) => + val value = v.getOrElse("") + s"${k}=$value" + } + .mkString("\n") +} +``` + + +## Search Api + +Provides http endpoints for searching the index. There is a [query +dsl](/docs/query-manual.md) for more convenient searching for renku entities. +Additionally, there is an openapi documentation generated and a version +endpoint. + +```scala mdoc:passthrough +CodeBlock.plainLines( + s"GET /${SearchApiMS.pathPrefix.mkString("/")}/query?q=", + s"GET /${SearchApiMS.pathPrefix.mkString("/")}/version", + s"GET /${SearchApiMS.pathPrefix.mkString("/")}/spec.json" +) +``` + +Here is an example of the result structure. For more details, the openapi doc +should be consulted. + +```scala mdoc:passthrough +JsonPrinter.block { + data.SearchResult( + items = List(tapir.ApiSchema.exampleProject, tapir.ApiSchema.exampleUser, tapir.ApiSchema.exampleGroup), + facets = data.FacetData(Map( + EntityType.Project -> 1, + EntityType.Group -> 1, + EntityType.User -> 1 + )), + pagingInfo = data.PageWithTotals(data.PageDef.default, 3L, false) + ) +} +``` + +### Configuration + +The service is configured via environment variables. Each variable is prefixed +with `RS_` (for "renku search"). + +```scala mdoc:passthrough +CodeBlock.plain { + SearchApiConfig.configValues + .getAll.toList.sortBy(_._1).map { case (k, v) => + val value = v.getOrElse("") + s"${k}=$value" + } + .mkString("\n") +} +``` diff --git a/flake.nix b/flake.nix index 0f3e9ad3..d1b36697 100644 --- a/flake.nix +++ b/flake.nix @@ -55,30 +55,29 @@ ++ (builtins.attrValues selfPkgs); queueNames = { - projectCreated = "project.created"; - projectUpdated = "project.updated"; - projectRemoved = "project.removed"; - projectAuthAdded = "projectAuth.added"; - projectAuthUpdated = "projectAuth.updated"; - projectAuthRemoved = "projectAuth.removed"; - userAdded = "user.added"; - userUpdated = "user.updated"; - userRemoved = "user.removed"; - groupAdded = "group.added"; - groupUpdated = "group.updated"; - groupRemoved = "groupRemoved"; - groupMemberAdded = "groupMemberAdded"; - groupMemberUpdated = "groupMemberUpdated"; - groupMemberRemoved = "groupMemberRemoved"; - dataServiceAllEvents = "data_service.all_events"; + PROJECT_CREATED = "project.created"; + PROJECT_UPDATED = "project.updated"; + PROJECT_REMOVED = "project.removed"; + PROJECTAUTH_ADDED = "projectAuth.added"; + PROJECTAUTH_UPDATED = "projectAuth.updated"; + PROJECTAUTH_REMOVED = "projectAuth.removed"; + USER_ADDED = "user.added"; + USER_UPDATED = "user.updated"; + USER_REMOVED = "user.removed"; + GROUP_ADDED = "group.added"; + GROUP_UPDATED = "group.updated"; + GROUP_REMOVED = "groupRemoved"; + GROUPMEMBER_ADDED = "groupMemberAdded"; + GROUPMEMBER_UPDATED = "groupMemberUpdated"; + GROUPMEMBER_REMOVED = "groupMemberRemoved"; + DATASERVICE_ALLEVENTS = "data_service.all_events"; }; queueNameConfig = with nixpkgs.lib; mapAttrs' (key: qn: nameValuePair "RS_REDIS_QUEUE_${key}" qn) queueNames; in { formatter = pkgs.alejandra; - devShells = rec { - default = container; + devShells = { container = pkgs.mkShellNoCC (queueNameConfig // { RS_SOLR_HOST = "rsdev-cnt"; @@ -136,6 +135,13 @@ commonPackages ++ (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts); }); + + ci = pkgs.mkShellNoCC { + buildInputs = [ + devshellToolsPkgs.sbt17 + ]; + SBT_OPTS = "-Xmx2G"; + }; }; }); } diff --git a/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala index c71e3797..238a64c7 100644 --- a/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala +++ b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala @@ -29,79 +29,92 @@ import io.renku.solr.client.{SolrConfig, SolrUser} import org.http4s.Uri import scala.concurrent.duration.* - -object ConfigValues extends ConfigDecoders: - - private val prefix = "RS" - - private def renv(name: String) = - env(s"${prefix}_$name") - - val logLevel: ConfigValue[Effect, Int] = - renv("LOG_LEVEL").default("2").as[Int] - - val redisConfig: ConfigValue[Effect, RedisConfig] = { - val host = renv("REDIS_HOST").default("localhost").as[RedisHost] - val port = renv("REDIS_PORT").default("6379").as[RedisPort] - val sentinel = renv("REDIS_SENTINEL").as[Boolean].default(false) - val maybeDB = renv("REDIS_DB").as[RedisDB].option - val maybePass = renv("REDIS_PASSWORD").as[RedisPassword].option - val maybeMasterSet = renv("REDIS_MASTER_SET").as[RedisMasterSet].option +import java.util.concurrent.atomic.AtomicReference + +final class ConfigValues(prefix: String = "RS") extends ConfigDecoders: + private val values = new AtomicReference[Map[String, Option[String]]](Map.empty) + + private def config( + name: String, + default: Option[String] + ): ConfigValue[Effect, String] = { + val fullName = s"${prefix}_${name.toUpperCase}" + values.updateAndGet(m => m.updated(fullName, default)) + val propName = fullName.toLowerCase.replace('_', '.') + val cv = prop(propName).or(env(fullName)) + default.map(cv.default(_)).getOrElse(cv) + } + private def config(name: String): ConfigValue[Effect, String] = config(name, None) + private def config(name: String, defval: String): ConfigValue[Effect, String] = + config(name, Some(defval)) + + def getAll: Map[String, Option[String]] = values.get() + + lazy val logLevel: ConfigValue[Effect, Int] = + config("LOG_LEVEL", "2").as[Int] + + lazy val redisConfig: ConfigValue[Effect, RedisConfig] = { + val host = config("REDIS_HOST", "localhost").as[RedisHost] + val port = config("REDIS_PORT", "6379").as[RedisPort] + val sentinel = config("REDIS_SENTINEL").as[Boolean].default(false) + val maybeDB = config("REDIS_DB").as[RedisDB].option + val maybePass = config("REDIS_PASSWORD").as[RedisPassword].option + val maybeMasterSet = config("REDIS_MASTER_SET").as[RedisMasterSet].option val connectionRefresh = - renv("REDIS_CONNECTION_REFRESH_INTERVAL").as[FiniteDuration].default(30 minutes) + config("REDIS_CONNECTION_REFRESH_INTERVAL", "30 minutes").as[FiniteDuration] (host, port, sentinel, maybeDB, maybePass, maybeMasterSet, connectionRefresh) .mapN(RedisConfig.apply) } def eventQueue(eventType: String): ConfigValue[Effect, QueueName] = - renv(s"REDIS_QUEUE_$eventType").as[QueueName] + config(s"REDIS_QUEUE_$eventType").as[QueueName] - val retryOnErrorDelay: ConfigValue[Effect, FiniteDuration] = - renv("RETRY_ON_ERROR_DELAY").default("10 seconds").as[FiniteDuration] + lazy val retryOnErrorDelay: ConfigValue[Effect, FiniteDuration] = + config("RETRY_ON_ERROR_DELAY", "10 seconds").as[FiniteDuration] - val metricsUpdateInterval: ConfigValue[Effect, FiniteDuration] = - renv("METRICS_UPDATE_INTERVAL").default("15 seconds").as[FiniteDuration] + lazy val metricsUpdateInterval: ConfigValue[Effect, FiniteDuration] = + config("METRICS_UPDATE_INTERVAL", "15 seconds").as[FiniteDuration] def clientId(default: ClientId): ConfigValue[Effect, ClientId] = - renv("CLIENT_ID").default(default.value).as[ClientId] + config("CLIENT_ID", default.value).as[ClientId] - val solrConfig: ConfigValue[Effect, SolrConfig] = { - val url = renv("SOLR_URL").default("http://localhost:8983").as[Uri] - val core = renv("SOLR_CORE").default("search-core-test") + lazy val solrConfig: ConfigValue[Effect, SolrConfig] = { + val url = config("SOLR_URL", "http://localhost:8983").as[Uri] + val core = config("SOLR_CORE", "search-core-test") val maybeUser = - (renv("SOLR_USER").default("admin"), renv("SOLR_PASS")) + (config("SOLR_USER", "admin"), config("SOLR_PASS")) .mapN(SolrUser.apply) .option val logMessageBodies = - renv("SOLR_LOG_MESSAGE_BODIES").default("false").as[Boolean] + config("SOLR_LOG_MESSAGE_BODIES", "false").as[Boolean] (url, core, maybeUser, logMessageBodies).mapN(SolrConfig.apply) } def httpServerConfig( - prefix: String, + serviceName: String, defaultPort: Port ): ConfigValue[Effect, HttpServerConfig] = val bindAddress = - renv(s"${prefix}_HTTP_SERVER_BIND_ADDRESS").default("0.0.0.0").as[Ipv4Address] + config(s"${serviceName.toUpperCase}_HTTP_SERVER_BIND_ADDRESS", "0.0.0.0").as[Ipv4Address] val port = - renv(s"${prefix}_HTTP_SERVER_PORT").default(defaultPort.value.toString).as[Port] + config(s"${serviceName.toUpperCase}_HTTP_SERVER_PORT", defaultPort.value.toString).as[Port] val shutdownTimeout = - renv(s"${prefix}_HTTP_SHUTDOWN_TIMEOUT").default("30s").as[Duration] + config("HTTP_SHUTDOWN_TIMEOUT", "30s").as[Duration] (bindAddress, port, shutdownTimeout).mapN(HttpServerConfig.apply) - val jwtVerifyConfig: ConfigValue[Effect, JwtVerifyConfig] = { + lazy val jwtVerifyConfig: ConfigValue[Effect, JwtVerifyConfig] = { val defaults = JwtVerifyConfig.default - val enableSigCheck = renv("JWT_ENABLE_SIGNATURE_CHECK") - .as[Boolean] - .default(defaults.enableSignatureValidation) - val requestDelay = renv("JWT_KEYCLOAK_REQUEST_DELAY") - .as[FiniteDuration] - .default(defaults.minRequestDelay) + val enableSigCheck = + config("JWT_ENABLE_SIGNATURE_CHECK", defaults.enableSignatureValidation.toString) + .as[Boolean] + val requestDelay = + config("JWT_KEYCLOAK_REQUEST_DELAY", defaults.minRequestDelay.toString) + .as[FiniteDuration] val openIdConfigPath = - renv("JWT_OPENID_CONFIG_PATH").default(defaults.openIdConfigPath) + config("JWT_OPENID_CONFIG_PATH", defaults.openIdConfigPath) val allowedIssuers = - renv("JWT_ALLOWED_ISSUER_URL_PATTERNS") + config("JWT_ALLOWED_ISSUER_URL_PATTERNS") .as[List[UrlPattern]] (requestDelay, enableSigCheck, openIdConfigPath, allowedIssuers).mapN( JwtVerifyConfig.apply diff --git a/modules/config-values/src/main/scala/io/renku/search/config/QueuesConfig.scala b/modules/config-values/src/main/scala/io/renku/search/config/QueuesConfig.scala index ee20f57c..b85f2191 100644 --- a/modules/config-values/src/main/scala/io/renku/search/config/QueuesConfig.scala +++ b/modules/config-values/src/main/scala/io/renku/search/config/QueuesConfig.scala @@ -60,22 +60,22 @@ final case class QueuesConfig( ) object QueuesConfig: - val config: ConfigValue[Effect, QueuesConfig] = + def config(cv: ConfigValues): ConfigValue[Effect, QueuesConfig] = ( - ConfigValues.eventQueue("projectCreated"), - ConfigValues.eventQueue("projectUpdated"), - ConfigValues.eventQueue("projectRemoved"), - ConfigValues.eventQueue("projectAuthAdded"), - ConfigValues.eventQueue("projectAuthUpdated"), - ConfigValues.eventQueue("projectAuthRemoved"), - ConfigValues.eventQueue("userAdded"), - ConfigValues.eventQueue("userUpdated"), - ConfigValues.eventQueue("userRemoved"), - ConfigValues.eventQueue("groupAdded"), - ConfigValues.eventQueue("groupUpdated"), - ConfigValues.eventQueue("groupRemoved"), - ConfigValues.eventQueue("groupMemberAdded"), - ConfigValues.eventQueue("groupMemberUpdated"), - ConfigValues.eventQueue("groupMemberRemoved"), - ConfigValues.eventQueue("dataServiceAllEvents") + cv.eventQueue("project_created"), + cv.eventQueue("project_updated"), + cv.eventQueue("project_removed"), + cv.eventQueue("projectauth_added"), + cv.eventQueue("projectauth_updated"), + cv.eventQueue("projectauth_removed"), + cv.eventQueue("user_added"), + cv.eventQueue("user_updated"), + cv.eventQueue("user_removed"), + cv.eventQueue("group_added"), + cv.eventQueue("group_updated"), + cv.eventQueue("group_removed"), + cv.eventQueue("groupmember_added"), + cv.eventQueue("groupmember_updated"), + cv.eventQueue("groupmember_removed"), + cv.eventQueue("dataservice_allevents") ).mapN(QueuesConfig.apply) diff --git a/modules/http4s-commons/src/main/scala/io/renku/search/http/routes/OperationRoutes.scala b/modules/http4s-commons/src/main/scala/io/renku/search/http/routes/OperationRoutes.scala index 049d3633..5124d579 100644 --- a/modules/http4s-commons/src/main/scala/io/renku/search/http/routes/OperationRoutes.scala +++ b/modules/http4s-commons/src/main/scala/io/renku/search/http/routes/OperationRoutes.scala @@ -27,9 +27,16 @@ import io.renku.search.common.CurrentVersion import io.renku.search.http.borer.TapirBorerJson object OperationRoutes extends TapirBorerJson { + enum Paths { + case Ping + case Version + + lazy val name: String = productPrefix.toLowerCase() + } + def pingEndpoint[F[_]: Async] = endpoint.get - .in("ping") + .in(Paths.Ping.name) .out(stringBody) .description("Ping") .serverLogicSuccess[F](_ => "pong".pure[F]) @@ -38,7 +45,7 @@ object OperationRoutes extends TapirBorerJson { def versionEndpoint[F[_]: Async] = endpoint.get - .in("version") + .in(Paths.Version.name) .out(borerJsonBody[CurrentVersion]) .description("Return version information") .serverLogicSuccess[F](_ => CurrentVersion.get.pure[F]) diff --git a/modules/readme/src/main/scala/io/renku/search/readme/BulletPoints.scala b/modules/readme/src/main/scala/io/renku/search/readme/BulletPoints.scala new file mode 100644 index 00000000..3ea53b67 --- /dev/null +++ b/modules/readme/src/main/scala/io/renku/search/readme/BulletPoints.scala @@ -0,0 +1,6 @@ +package io.renku.search.readme + +object BulletPoints: + + def apply(lines: Seq[String]): Unit = + println(lines.mkString("- ", "\n- ", "")) diff --git a/modules/readme/src/main/scala/io/renku/search/readme/CodeBlock.scala b/modules/readme/src/main/scala/io/renku/search/readme/CodeBlock.scala new file mode 100644 index 00000000..cae153ac --- /dev/null +++ b/modules/readme/src/main/scala/io/renku/search/readme/CodeBlock.scala @@ -0,0 +1,12 @@ +package io.renku.search.readme + +object CodeBlock: + def plainLines(lines: String*): Unit = + plain(lines.mkString("\n")) + + def plain(content: String): Unit = apply(content, "") + def lang(lang: String)(content: String): Unit = apply(content, lang) + def apply(content: String, lang: String = ""): Unit = + println(s"``` $lang") + println(content) + println("```") diff --git a/modules/readme/src/main/scala/io/renku/search/readme/JsonPrinter.scala b/modules/readme/src/main/scala/io/renku/search/readme/JsonPrinter.scala new file mode 100644 index 00000000..0201204e --- /dev/null +++ b/modules/readme/src/main/scala/io/renku/search/readme/JsonPrinter.scala @@ -0,0 +1,45 @@ +package io.renku.search.readme + +import io.bullet.borer.{Encoder, Json} + +object JsonPrinter: + + def block[A: Encoder](value: A): Unit = + CodeBlock.lang("json")( + pretty(Json.encode(value).toUtf8String) + ) + + def str[A: Encoder](value: A) = pretty(Json.encode(value).toUtf8String) + + def pretty(json: String): String = pp(json) + + // yes, it is ugly, but only for the readme and avoids yet another library + @annotation.tailrec + private def pp( + in: String, + depth: Int = 0, + inQuote: Boolean = false, + res: String = "" + ): String = + def spnl(n: Int) = "\n" + List.fill(n)(" ").mkString + in.headOption match { + case None => res + case Some('"') => + pp(in.drop(1), depth, !inQuote, res + '"') + case Some(' ') => + val next = if (inQuote) res + ' ' else res + pp(in.drop(1), depth, inQuote, next) + case Some(c) if c == '{' || c == '[' => + val next = res + c + spnl((depth + 1) * 2) + pp(in.drop(1), depth + 1, inQuote, next) + case Some(c) if c == '}' || c == ']' => + val next = res + spnl((depth - 1) * 2) + c + pp(in.drop(1), depth - 1, inQuote, next) + case Some(':') => + pp(in.drop(1), depth, inQuote, res + ": ") + case Some(',') => + val next = res + ',' + Option.when(!inQuote)(spnl(depth * 2)).getOrElse("") + pp(in.drop(1), depth, inQuote, next) + case Some(c) => + pp(in.drop(1), depth, inQuote, res + c) + } diff --git a/modules/readme/src/main/scala/io/renku/search/readme/package.scala b/modules/readme/src/main/scala/io/renku/search/readme/package.scala new file mode 100644 index 00000000..590f2164 --- /dev/null +++ b/modules/readme/src/main/scala/io/renku/search/readme/package.scala @@ -0,0 +1,8 @@ +package io.renku.search + +package object readme { + + extension (self: String) + def backticks: String = s"`${self}`" + def quoted: String = s"\"${self}\"" +} diff --git a/modules/redis-client/src/main/scala/io/renku/redis/client/MessageBodyKeys.scala b/modules/redis-client/src/main/scala/io/renku/redis/client/MessageBodyKeys.scala index 97598827..5a6f986f 100644 --- a/modules/redis-client/src/main/scala/io/renku/redis/client/MessageBodyKeys.scala +++ b/modules/redis-client/src/main/scala/io/renku/redis/client/MessageBodyKeys.scala @@ -18,6 +18,8 @@ package io.renku.redis.client -private object MessageBodyKeys: - val headers = "headers" - val payload = "payload" +enum MessageBodyKeys: + case Headers + case Payload + + lazy val name: String = productPrefix.toLowerCase() diff --git a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisQueueClient.scala b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisQueueClient.scala index c3e3cdbf..3d1d88b2 100644 --- a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisQueueClient.scala +++ b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisQueueClient.scala @@ -93,8 +93,8 @@ class RedisQueueClientImpl[F[_]: Async: Log](client: RedisClient) payload: ByteVector ): F[MessageId] = val messageBody = Map( - MessageBodyKeys.headers -> header, - MessageBodyKeys.payload -> payload + MessageBodyKeys.Headers.name -> header, + MessageBodyKeys.Payload.name -> payload ) val message = Stream.emit[F, XAddMessage[String, ByteVector]]( XAddMessage(queueName.name, messageBody) @@ -117,13 +117,16 @@ class RedisQueueClientImpl[F[_]: Async: Log](client: RedisClient) .getOrElse(StreamingOffset.All[String]) def toMessage(rm: XReadMessage[String, ByteVector]): Option[RedisMessage] = - (rm.body.get(MessageBodyKeys.headers), rm.body.get(MessageBodyKeys.payload)) + ( + rm.body.get(MessageBodyKeys.Headers.name), + rm.body.get(MessageBodyKeys.Payload.name) + ) .mapN(RedisMessage(rm.id.value, _, _)) lazy val logInfo: ((XReadMessage[?, ?], Option[RedisMessage])) => F[Unit] = { case (m, None) => logger.warn( - s"Message '${m.id}' skipped as it has no '${MessageBodyKeys.headers}' or '${MessageBodyKeys.payload}'" + s"Message '${m.id}' skipped as it has no '${MessageBodyKeys.Headers}' or '${MessageBodyKeys.Payload}'" ) case _ => ().pure[F] } diff --git a/modules/redis-client/src/test/scala/io/renku/redis/client/RedisQueueClientSpec.scala b/modules/redis-client/src/test/scala/io/renku/redis/client/RedisQueueClientSpec.scala index 534d0bc5..a080fa48 100644 --- a/modules/redis-client/src/test/scala/io/renku/redis/client/RedisQueueClientSpec.scala +++ b/modules/redis-client/src/test/scala/io/renku/redis/client/RedisQueueClientSpec.scala @@ -207,7 +207,7 @@ class RedisQueueClientSpec extends CatsEffectSuite with RedisBaseSuite: val message = Stream.emit[IO, XAddMessage[String, ByteVector]]( XAddMessage( queueName.name, - Map(MessageBodyKeys.payload -> payload) + Map(MessageBodyKeys.Payload.name -> payload) ) ) makeStreamingConnection(client) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala index e72c7327..795520ea 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala @@ -35,10 +35,18 @@ final case class SearchApiConfig( ) object SearchApiConfig: + private val configKeys = { + val cv = ConfigValues() + cv -> ( + cv.solrConfig, + cv.httpServerConfig("SEARCH", port"8080"), + cv.jwtVerifyConfig, + cv.logLevel + ) + } + + val configValues = configKeys._1 + val config: ConfigValue[Effect, SearchApiConfig] = - ( - ConfigValues.solrConfig, - ConfigValues.httpServerConfig("SEARCH", port"8080"), - ConfigValues.jwtVerifyConfig, - ConfigValues.logLevel - ).mapN(SearchApiConfig.apply) + val (_, keys) = configKeys + keys.mapN(SearchApiConfig.apply) diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/Services.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/Services.scala index c0624f17..ffd529dd 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/Services.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/Services.scala @@ -23,6 +23,7 @@ import cats.effect.* import io.renku.queue.client.QueueClient import io.renku.redis.client.ClientId import io.renku.search.config.ConfigValues +import io.renku.search.config.QueuesConfig import io.renku.search.events.* object Services: @@ -30,8 +31,11 @@ object Services: private val clientId: ClientId = ClientId("search-provision") private val counter = Ref.unsafe[IO, Long](0) + val configValues = ConfigValues() + val queueConfig = QueuesConfig.config(configValues) + def queueClient: Resource[IO, QueueClient[IO]] = - val redisCfg = ConfigValues.redisConfig.load[IO] + val redisCfg = ConfigValues().redisConfig.load[IO] Resource.eval(redisCfg).flatMap(QueueClient.make[IO](_, clientId)) private def makeRequestId: IO[RequestId] = diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/AddCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/AddCmd.scala index fe7993b3..8192b98f 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/AddCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/AddCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupAdded import io.renku.search.model.* @@ -38,7 +37,7 @@ object AddCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberAddCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberAddCmd.scala index 72aae611..7de09ee0 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberAddCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberAddCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupMemberAdded import io.renku.search.model.* @@ -38,7 +37,7 @@ object MemberAddCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberRemoveCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberRemoveCmd.scala index 02456116..77fea392 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberRemoveCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberRemoveCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupMemberRemoved import io.renku.search.model.* @@ -38,7 +37,7 @@ object MemberRemoveCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberUpdateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberUpdateCmd.scala index 661a8942..597ee6a5 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberUpdateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/MemberUpdateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupMemberUpdated import io.renku.search.model.* @@ -38,7 +37,7 @@ object MemberUpdateCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/RemoveCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/RemoveCmd.scala index 66b62997..51cd9af2 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/RemoveCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/RemoveCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupRemoved import io.renku.search.model.* @@ -38,7 +37,7 @@ object RemoveCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/UpdateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/UpdateCmd.scala index 54b96252..a945fd71 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/groups/UpdateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/groups/UpdateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.GroupUpdated import io.renku.search.model.* @@ -38,7 +37,7 @@ object UpdateCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/CreateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/CreateCmd.scala index ed96651c..ca59c87a 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/CreateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/CreateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectCreated import io.renku.search.model.* @@ -71,7 +70,7 @@ object CreateCmd extends CommonOpts: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberAddCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberAddCmd.scala index a0cd9d3b..2ec417f3 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberAddCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberAddCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectMemberAdded import io.renku.search.model.* @@ -39,7 +38,7 @@ object MemberAddCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberRemoveCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberRemoveCmd.scala index f4940776..4c4a736b 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberRemoveCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberRemoveCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectMemberRemoved import io.renku.search.model.* @@ -38,7 +37,7 @@ object MemberRemoveCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberUpdateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberUpdateCmd.scala index d2741c35..239c48aa 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberUpdateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/MemberUpdateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectMemberUpdated import io.renku.search.model.* @@ -40,7 +39,7 @@ object MemberUpdateCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/RemoveCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/RemoveCmd.scala index 428bc2e0..32d89cbe 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/RemoveCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/RemoveCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectRemoved import io.renku.search.model.* @@ -38,7 +37,7 @@ object RemoveCmd extends CommonOpts: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/UpdateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/UpdateCmd.scala index c9818333..6eb5117f 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/projects/UpdateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/projects/UpdateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ProjectUpdated import io.renku.search.model.* @@ -65,7 +64,7 @@ object UpdateCmd extends CommonOpts: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/FinishCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/FinishCmd.scala index 766268e4..5558422b 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/FinishCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/FinishCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ReprovisioningFinished import io.renku.search.model.* @@ -38,7 +37,7 @@ object FinishCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/StartCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/StartCmd.scala index f9ac5e84..ab9bc41f 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/StartCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/reprovision/StartCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.ReprovisioningStarted import io.renku.search.model.* @@ -38,7 +37,7 @@ object StartCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/users/AddCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/users/AddCmd.scala index 92ce14a1..425c8b43 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/users/AddCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/users/AddCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.UserAdded import io.renku.search.model.* @@ -50,7 +49,7 @@ object AddCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/users/RemoveCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/users/RemoveCmd.scala index 80a29fd9..1b7c0f5a 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/users/RemoveCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/users/RemoveCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.UserRemoved import io.renku.search.model.* @@ -38,7 +37,7 @@ object RemoveCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-cli/src/main/scala/io/renku/search/cli/users/UpdateCmd.scala b/modules/search-cli/src/main/scala/io/renku/search/cli/users/UpdateCmd.scala index 2d193462..152c17aa 100644 --- a/modules/search-cli/src/main/scala/io/renku/search/cli/users/UpdateCmd.scala +++ b/modules/search-cli/src/main/scala/io/renku/search/cli/users/UpdateCmd.scala @@ -23,7 +23,6 @@ import cats.syntax.all.* import com.monovore.decline.Opts import io.renku.search.cli.{CommonOpts, Services} -import io.renku.search.config.QueuesConfig import io.renku.search.events.UserUpdated import io.renku.search.model.* @@ -50,7 +49,7 @@ object UpdateCmd: def apply(cfg: Options): IO[ExitCode] = Services.queueClient.use { queue => for - queuesCfg <- QueuesConfig.config.load[IO] + queuesCfg <- Services.queueConfig.load[IO] msg <- Services.createMessage(cfg.asPayload) _ <- queue.enqueue(queuesCfg.dataServiceAllEvents, msg) yield ExitCode.Success diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Routes.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Routes.scala index 627819f9..03fc6433 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Routes.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Routes.scala @@ -23,6 +23,7 @@ import cats.syntax.all.* import fs2.io.net.Network import io.bullet.borer.Decoder +import io.bullet.borer.Encoder import io.bullet.borer.derivation.MapBasedCodecs import io.renku.search.events.MessageId import io.renku.search.http.borer.BorerEntityJsonCodec @@ -34,7 +35,7 @@ import org.http4s.HttpRoutes import org.http4s.dsl.Http4sDsl import org.http4s.server.Router -final private class Routes[F[_]: Async]( +final class Routes[F[_]: Async]( metricsRoutes: HttpRoutes[F], services: Services[F] ) extends Http4sDsl[F] @@ -58,7 +59,12 @@ final private class Routes[F[_]: Async]( } } -private object Routes: +object Routes: + enum Paths { + case Reindex + lazy val name: String = productPrefix.toLowerCase() + } + def apply[F[_]: Async: Network]( registryBuilder: CollectorRegistryBuilder[F], services: Services[F] @@ -72,3 +78,4 @@ private object Routes: .getOrElse(ReprovisionRequest.lastStart) object ReIndexMessage: given Decoder[ReIndexMessage] = MapBasedCodecs.deriveDecoder + given Encoder[ReIndexMessage] = MapBasedCodecs.deriveEncoder diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala index b1acdaf8..b69d03df 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala @@ -42,14 +42,22 @@ final case class SearchProvisionConfig( object SearchProvisionConfig: + private val configKeys = { + val cv = ConfigValues() + cv -> ( + cv.redisConfig, + cv.solrConfig, + cv.retryOnErrorDelay, + cv.metricsUpdateInterval, + cv.logLevel, + QueuesConfig.config(cv), + cv.httpServerConfig("PROVISION", defaultPort = port"8081"), + cv.clientId(ClientId("search-provisioner")) + ) + } + + val configValues = configKeys._1 + val config: ConfigValue[Effect, SearchProvisionConfig] = - ( - ConfigValues.redisConfig, - ConfigValues.solrConfig, - ConfigValues.retryOnErrorDelay, - ConfigValues.metricsUpdateInterval, - ConfigValues.logLevel, - QueuesConfig.config, - ConfigValues.httpServerConfig("PROVISION", defaultPort = port"8081"), - ConfigValues.clientId(ClientId("search-provisioner")) - ).mapN(SearchProvisionConfig.apply) + val (_, keys) = configKeys + keys.mapN(SearchProvisionConfig.apply) diff --git a/modules/search-query-docs/docs/manual.md b/modules/search-query-docs/docs/manual.md index aaeebb4f..d60649ae 100644 --- a/modules/search-query-docs/docs/manual.md +++ b/modules/search-query-docs/docs/manual.md @@ -1,8 +1,8 @@ + ## Search Query -The search accepts queries in two representations: JSON and a simple -query string. A query may contain specific and unspecific search -terms. +The search accepts queries as a query string. A query may contain +specific and unspecific search terms. ### Query String diff --git a/project/ReadmePlugin.scala b/project/ReadmePlugin.scala new file mode 100644 index 00000000..bbf654a7 --- /dev/null +++ b/project/ReadmePlugin.scala @@ -0,0 +1,135 @@ +import sbt._ +import sbt.Keys._ +import mdoc.MdocPlugin +import com.github.sbt.git.{GitPlugin, JGit} +import com.github.sbt.git.SbtGit.GitKeys +import org.eclipse.jgit.api._ +import org.eclipse.jgit.diff.DiffEntry +import org.eclipse.jgit.treewalk.CanonicalTreeParser +import scala.jdk.CollectionConverters._ + +object ReadmePlugin extends AutoPlugin { + override def requires = MdocPlugin && GitPlugin + object autoImport { + val readmeUpdate = inputKey[Unit]("Update the root README.md") + val readmeBaseRef = settingKey[String]("The base ref to check modification against") + val readmeCheckModification = taskKey[Unit]( + "Check the diff against the target branch for modifications to generated files" + ) + val readmeAdditionalFiles = settingKey[Map[File, String]]( + "Additional (generated) files to copy into the root docs/ folder" + ) + } + + import autoImport._ + import MdocPlugin.autoImport._ + + override def projectSettings = Seq( + libraryDependencies ++= Seq.empty, + mdocIn := (LocalRootProject / baseDirectory).value / "docs" / "readme.md", + mdocOut := (LocalRootProject / baseDirectory).value / "README.md", + fork := true, + readmeAdditionalFiles := Map.empty, + readmeCheckModification := { + val bref = readmeBaseRef.value + val dir = (LocalRootProject / baseDirectory).value + val additional = readmeAdditionalFiles.value + val readmeIn = mdocIn.value + val readmeOut = mdocOut.value + val logger = streams.value.log + checkModification(logger, bref, dir, readmeIn, readmeOut, additional, dir / "docs") + }, + readmeBaseRef := sys.env + .get("README_BASE_REF") + .map(_.trim) + .filter(_.nonEmpty) + .map(ref => s"${ref}^{tree}") + .getOrElse("HEAD^{tree}"), + readmeUpdate := { + mdoc.evaluated + val logger = streams.value.log + val additional = readmeAdditionalFiles.value + val addout = (LocalRootProject / baseDirectory).value / "docs" + additional.toList.foreach { case (src, name) => + val dest = addout / name + logger.info(s"Copying $src -> $dest") + IO.copyFile(src, dest) + } + () + } + ) + + def checkModification( + logger: Logger, + baseRef: String, + sourceRoot: File, + readmeIn: File, + readmeOut: File, + additional: Map[File, String], + additionalOut: File + ) = { + val git = JGit(sourceRoot).porcelain + val repo = git.getRepository + val base = repo.resolve(baseRef) + if (base eq null) sys.error(s"Cannot resolve base ref: $baseRef") + val p = new CanonicalTreeParser() + val reader = repo.newObjectReader() + p.reset(reader, base) + val diff = git + .diff() + .setOldTree(p) + .setShowNameOnly(true) + .call() + .asScala + .filter(e => e.getChangeType == DiffEntry.ChangeType.MODIFY) + .map(_.getOldPath) + .toSet + + logger.debug(s"Changed files from $baseRef: $diff") + + val checker = Checker(sourceRoot, diff, logger) + + // check readme + checker.check(readmeIn, readmeOut) + // check others + additional.toList.foreach { case (src, targetName) => + val dest = additionalOut / targetName + checker.check(src, dest) + } + () + } + + def relativize(base: File, file: File): String = + base + .relativize(file) + .map(_.toString) + .getOrElse( + sys.error( + s"Cannot obtain path into repository for $file" + ) + ) + + final case class Checker(sourceRoot: File, diff: Set[String], logger: Logger) { + def check(src: File, dest: File) = { + val srcPath = relativize(sourceRoot, src) + val destPath = relativize(sourceRoot, dest) + val srcModified = diff.contains(srcPath) + val destModified = diff.contains(destPath) + def modWord(modified: Boolean) = if (modified) "modified" else "not modified" + logger.debug( + s"${modWord(srcModified)} $srcPath | ${modWord(destModified)} $destPath" + ) + logger.info(s"Check modification $srcPath -> $destPath …") + if (srcModified && !destModified) { + logger.error(s"You changed $srcPath but did not generate the output file") + sys.error("Checking failed") + } + if (destModified && !srcModified) { + logger.error( + s"You changed $destPath but this is a generated file, please change $srcPath instead" + ) + sys.error("Checking failed") + } + } + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index cef66eff..f9a6cf78 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -25,4 +25,10 @@ addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") addSbtPlugin("com.julianpeeters" % "sbt-avrohugger" % "2.8.3") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") addSbtPlugin("io.spray" % "sbt-revolver" % "0.10.0") +addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.5.3") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") + +// sbt-git comes with quite old jgit version +libraryDependencies ++= Seq( + "org.eclipse.jgit" % "org.eclipse.jgit" % "7.0.0.202409031743-r" +)