feat: Update cat-gateway packaging, update README.md with guidance ho…

…w to locally spin up cat-gateway with event-db (#437) * update cat-gateway packaging * fix * fix * fix * fix * fix spelling * update entry.sh comments * wip * fix --------- Co-authored-by: Oleksandr Prokhorenko <[email protected]>
input-output-hk · Apr 19, 2024 · 3676b95 · 3676b95
1 parent f585c2f
commit 3676b95
Show file tree

Hide file tree

Showing 12 changed files with 139 additions and 132 deletions.
diff --git a/catalyst-gateway/Earthfile b/catalyst-gateway/Earthfile
@@ -31,8 +31,9 @@ build:
     FROM +builder
 
     TRY
-        RUN /scripts/std_build.py   --cov_report="coverage-report.info" \
-                                    --bins="cat-gateway/cat-gateway"
+        RUN /scripts/std_build.py \
+                --cov_report="coverage-report.info" \
+                --bins="cat-gateway/cat-gateway"
     FINALLY
         SAVE ARTIFACT --if-exists target/nextest/ci/junit.xml AS LOCAL cat-gateway.junit-report.xml
         SAVE ARTIFACT --if-exists coverage-report.info AS LOCAL cat-gateway.coverage-report.info
@@ -47,29 +48,29 @@ all-hosts-build:
     BUILD --platform=linux/amd64 --platform=linux/arm64 +build
 
 package-cat-gateway:
+    ARG tag="latest"
+
     FROM alpine:3.19
+    WORKDIR /cat-gateway
+
+    RUN apk add --no-cache gcc bash
 
+    COPY +build/cat-gateway .
+    COPY ./scripts/entry.sh .
+
+    ENTRYPOINT ./entry.sh
+    SAVE IMAGE cat-gateway:$tag
+
+package-cat-gateway-with-preprod-snapshot:
     ARG tag="latest"
-    ARG address
-    ARG db_url
-    ARG log_level="error"
 
-    RUN apk add --no-cache gcc
+    FROM +package-cat-gateway
 
-    COPY +build/cat-gateway .
+    # copy preprod mithril snapshot to /tmp/preprod dir
+    COPY github.com/input-output-hk/catalyst-ci/earthly/mithril_snapshot:v2.11.1+package-preprod-snapshot/snapshot /tmp/preprod
 
-    ENTRYPOINT ./cat-gateway run --address $address --database-url $db_url --log-level $log_level
     SAVE IMAGE cat-gateway:$tag
 
 # Publish packages if all integration tests have passed. (Failure to pass tests will prevent packages being published.)
 # publish:
 #    FROM scratch
-
-## -----------------------------------------------------------------------------
-
-# This step simulates the full CI run for local purposes only.
-local-ci-run:
-    BUILD +check
-    BUILD +build
-    # BUILD +package
-    # BUILD +publish
diff --git a/catalyst-gateway/README.md b/catalyst-gateway/README.md
@@ -5,6 +5,9 @@
     * [`./bin`](#bin)
     * [`./crates`](#crates)
     * [`./event-db`](#event-db)
+  * [Build and Run](#build-and-run)
+    * [Docker images](#docker-images)
+    * [Rust binary](#rust-binary)
 
 The Catalyst Data Gateway is the backend of the Catalyst Voices hosted stack.
 
@@ -26,3 +29,42 @@ They are also able to be used stand-alone in other projects and can be published
 
 Defines the Postgres Catalyst Event Database that the Catalyst gateway uses for running Catalyst Events.
 This is DB definition and data only, the actual db interface code is located at `./bin/src/event-db`.
+
+## Build and Run
+
+There are several options how you can build this service,
+as a Rust binary from the Rust code explicitly,
+or you can build a docker image and run everything with the `docker-compose`.
+
+### Docker images
+
+To build and run docker images follow these steps:
+
+1. Run `earthly +package-cat-gateway` or `earthly +package-cat-gateway-with-preprod-snapshot`
+  to build a cat-gateway docker image without `preprod-snapshot` or with it.
+2. Run `earthly ./event-db+build` to build an event-db docker image.
+3. Run `docker-compose up cat-gateway` to spin up cat-gateway with event-db from already built images.
+
+Note that every time when you are building an image it obsoletes an old image but does not remove it,
+so dont forget to cleanup dangling images of the event-db and cat-gateway in your docker environment.
+
+### Rust binary
+
+To build and run a Rust binary follow these steps:
+
+1. Run `cargo build -p cat-gateway --release`
+  to compile a release version of the cat-gateway
+2. Run `earthly ./event-db+build` to build an event-db docker image
+3. If you need to have a `preprod-snapshot` unarchive snapshot data to the `/tmp/preprod/` dir.
+  You can download `preprod-snapshot` from this
+  [resource](https://mithril.network/explorer/?aggregator=https%3A%2F%2Faggregator.release-preprod.api.mithril.network%2Faggregator).
+4. Run
+
+    ```sh
+      ./target/release/cat-gateway run \
+      --address "127.0.0.1:3030" \
+      --database-url=postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev \
+      --log-level=debug \
+      --log-format=compact \
+      --metrics-address "127.0.0.1:3032"
+    ```
diff --git a/catalyst-gateway/bin/src/service/api/mod.rs b/catalyst-gateway/bin/src/service/api/mod.rs
@@ -93,48 +93,32 @@ pub(crate) fn mk_api(
 
     // Get localhost name
     if let Ok(hostname) = gethostname().into_string() {
-        let hostname_addresses = add_protocol_prefix(
-            &format!("{hostname}:{port}"),
-            settings.http_auto_servers,
-            settings.https_auto_servers,
-        );
-        for hostname_address in hostname_addresses {
-            service = service.server(
-                ServerObject::new(hostname_address).description("Server at localhost name"),
-            );
-        }
+        let hostname_address = format!("http://{hostname}:{port}");
+        service = service
+            .server(ServerObject::new(hostname_address).description("Server at localhost name"));
     }
 
     // Get local IP address v4 and v6
     if let Ok(network_interfaces) = list_afinet_netifas() {
         for (name, ip) in &network_interfaces {
             if *name == "en0" {
-                let (ip_with_port, desc) = match ip {
-                    IpAddr::V4(_) => (format!("{ip}:{port}"), "Server at local IPv4 address"),
-                    IpAddr::V6(_) => (format!("[{ip}]:{port}"), "Server at local IPv6 address"),
+                let (address, desc) = match ip {
+                    IpAddr::V4(_) => {
+                        (
+                            format!("http://{ip}:{port}"),
+                            "Server at local IPv4 address",
+                        )
+                    },
+                    IpAddr::V6(_) => {
+                        (
+                            format!("http://[{ip}]:{port}"),
+                            "Server at local IPv6 address",
+                        )
+                    },
                 };
-                let ip_addresses = add_protocol_prefix(
-                    &ip_with_port,
-                    settings.http_auto_servers,
-                    settings.https_auto_servers,
-                );
-                for address in ip_addresses {
-                    service = service.server(ServerObject::new(address).description(desc));
-                }
+                service = service.server(ServerObject::new(address).description(desc));
             }
         }
     }
     service
 }
-
-/// Function to add protocol prefix based on flags.
-fn add_protocol_prefix(address: &String, is_http: bool, is_https: bool) -> Vec<String> {
-    let mut addresses = Vec::new();
-    if is_http {
-        addresses.push(format!("http://{address}"));
-    }
-    if is_https {
-        addresses.push(format!("https://{address}"));
-    }
-    addresses
-}
diff --git a/catalyst-gateway/bin/src/settings.rs b/catalyst-gateway/bin/src/settings.rs
@@ -79,14 +79,6 @@ pub(crate) struct DocsSettings {
     #[clap(long, default_value = ADDRESS_DEFAULT, env = "ADDRESS")]
     pub(crate) address: SocketAddr,
 
-    /// Flag for adding "http" to servers
-    #[clap(long, default_value = "false", env = "HTTP_AUTO_SERVERS")]
-    pub(crate) http_auto_servers: bool,
-
-    /// Flag for adding "https" to servers
-    #[clap(long, default_value = "true", env = "HTTPS_AUTO_SERVERS")]
-    pub(crate) https_auto_servers: bool,
-
     /// Server name
     #[clap(long, env = "SERVER_NAME")]
     pub(crate) server_name: Option<String>,

diff --git a/catalyst-gateway/docker-compose.yml b/catalyst-gateway/docker-compose.yml
@@ -0,0 +1,42 @@
+version: "3"
+
+services:
+  event-db:
+    image: event-db:latest
+    environment:
+      # Required environment variables for migrations
+      - DB_HOST=localhost
+      - DB_PORT=5432
+      - DB_NAME=CatalystEventDev
+      - DB_DESCRIPTION="Catalyst Event DB"
+      - DB_SUPERUSER=postgres
+      - DB_SUPERUSER_PASSWORD=postgres
+      - DB_USER=catalyst-event-dev
+      - DB_USER_PASSWORD=CHANGE_ME
+
+      - INIT_AND_DROP_DB=true
+      - WITH_MIGRATIONS=true
+      - WITH_SEED_DATA=true
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${DB_SUPERUSER} -d $${DB_SUPERUSER_PASSWORD}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  cat-gateway:
+    image: cat-gateway:latest
+    environment:
+      - DB_URL=postgres://catalyst-event-dev:CHANGE_ME@event-db/CatalystEventDev
+      - CAT_ADDRESS=0.0.0.0:3030
+      - LOG_LEVEL=info
+      - DATA_REFRESH_TICK=5
+      - CHECK_CONFIG_TICK=5
+      - MACHINE_ID="UID"
+    hostname: 127.0.0.1
+    ports:
+      - 3030:3030
+    depends_on:
+      event-db:
+        condition: service_healthy
diff --git a/catalyst-gateway/event-db/Readme.md b/catalyst-gateway/event-db/Readme.md
@@ -6,22 +6,8 @@ This crate defines the necessary migrations, seed data and docker image builder
 
 Firstly you will need to prepare a docker images with all migrations and data.
 
-Prepare a event-db docker image with the historic and test data
+Prepare and build an event-db docker image
 
 ```sh
-earthly ./catalyst-gateway/event-db+build
-```
-
-Run a event db docker container
-
-```sh
-docker-compose -f catalyst-gateway/event-db/docker-compose.yml up event-db
-```
-
-This will run postgres on port `5432`.
-
-To test that docker image builds fine and migrations correctly applies run
-
-```sh
-earthly -P ./catalyst-gateway/event-db+test
+earthly +build
 ```
diff --git a/catalyst-gateway/event-db/docker-compose.yml b/catalyst-gateway/event-db/docker-compose.yml
diff --git a/catalyst-gateway/bin/entry.sh → catalyst-gateway/scripts/entry.sh b/catalyst-gateway/bin/entry.sh → catalyst-gateway/scripts/entry.sh
@@ -5,26 +5,15 @@
 # ---------------------------------------------------------------
 #
 # This script serves as the entrypoint for the cat-gateway service container. 
-# It sets up the environment, performing optional database initialization 
-# if configured, and then runs the cat-gateway service.
 #
 # It expects the following environment variables to be set except where noted:
 #
-# DATABASE_URL - URL for the EventDB.
-# DEBUG - If set, the script will print debug information (optional)
+# DB_URL - URL for the EventDB.
+# CAT_ADDRESS - Address to bind the cat-gateway service to.
+# LOG_LEVEL - Log level for the cat-gateway service.
 # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional)
 # ---------------------------------------------------------------
 
-# Enable strict mode
-set +x
-set -o errexit
-set -o pipefail
-set -o nounset
-set -o functrace
-set -o errtrace
-set -o monitor
-set -o posix
-shopt -s dotglob
 
 check_env_vars() {
     local env_vars=("$@")
@@ -49,7 +38,9 @@ echo ">>> Starting entrypoint script..."
 
 # Check if all required environment variables are set
 REQUIRED_ENV=(
-    "DATABASE_URL"
+    "DB_URL"
+    "CAT_ADDRESS"
+    "LOG_LEVEL"
 )
 check_env_vars "${REQUIRED_ENV[@]}"
 
@@ -58,7 +49,7 @@ debug_sleep
 
 # Define the command to be executed
 ARGS=$*
-CMD="/app/cat-gateway run $ARGS"
+CMD="./cat-gateway run --address $CAT_ADDRESS --database-url $DB_URL --log-level $LOG_LEVEL $ARGS"
 echo ">>> Executing command: $CMD"
 
 # Wait for DEBUG_SLEEP seconds if the DEBUG_SLEEP environment variable is set
@@ -77,4 +68,4 @@ set -e
 if [ $EXIT_CODE -ne 0 ]; then
   echo "Error: cat-gateway returned with exit code $EXIT_CODE"
   exit 1
-fi
+fi
diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile
@@ -32,16 +32,14 @@ fuzzer-api:
     FROM earthly/dind:alpine-3.19
     RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
     RUN apk add yq zstd
-    ARG DB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev"
-    ARG CAT_ADDRESS="127.0.0.1:3030"
     ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json"
 
     COPY schemathesis-docker-compose.yml .
     WITH DOCKER \
         --compose schemathesis-docker-compose.yml \
         --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \
         --load event-db:latest=(../event-db+build) \
-        --load cat-gateway:latest=(../+package-cat-gateway --address=$CAT_ADDRESS --db_url=$DB_URL) \
+        --load cat-gateway:latest=(../+package-cat-gateway) \
         --service event-db \
         --service cat-gateway \
         --allow-privileged

diff --git a/catalyst-gateway/tests/api_tests/Earthfile b/catalyst-gateway/tests/api_tests/Earthfile
@@ -8,18 +8,6 @@ builder:
     COPY ./snapshot_tool-56364174.json .
     DO github.com/input-output-hk/catalyst-ci/earthly/python:v2.11.1+BUILDER
 
-cat-gateway-test:
-    ARG DB_URL="postgres://catalyst-event-dev:CHANGE_ME@event-db/CatalystEventDev"
-    ARG CAT_ADDRESS="0.0.0.0:3030"
-    ARG LOG_LEVEL="error"
-
-    FROM ../../+package-cat-gateway --address=$CAT_ADDRESS --db_url=$DB_URL --log_level=$LOG_LEVEL
-
-    # copy preprod mithril snapshot to /tmp/preprod dir
-    COPY github.com/input-output-hk/catalyst-ci/earthly/mithril_snapshot:v2.11.1+package-preprod-snapshot/snapshot /tmp/preprod
-
-    SAVE IMAGE cat-gateway-test:latest
-
 test:
     FROM +builder
 
@@ -28,8 +16,8 @@ test:
     WITH DOCKER \
         --compose docker-compose.yml \
         --load event-db:latest=(../../event-db+build) \
-        --load cat-gateway-test:latest=(+cat-gateway-test) \
+        --load cat-gateway:latest=(../../+package-cat-gateway-with-preprod-snapshot) \
         --service cat-gateway \
         --allow-privileged
         RUN poetry run pytest -s
-    END  
+    END