diff --git a/.github/workflows/portal-loop.yml b/.github/workflows/portal-loop.yml index 01135b164ac..b898a149e9d 100644 --- a/.github/workflows/portal-loop.yml +++ b/.github/workflows/portal-loop.yml @@ -57,13 +57,12 @@ jobs: - name: "Checkout" uses: actions/checkout@v4 - - name: "Setup the images" + - name: "Setup The portal loop docker compose" run: | cd misc/loop - - docker compose build - docker compose pull - docker compose up -d + echo "Making docker compose happy" + touch .env + make docker.ci - name: "Test1 - Portal loop start gnoland" run: | diff --git a/misc/loop/.env.example b/misc/loop/.env.example new file mode 100644 index 00000000000..af75eaa9fc7 --- /dev/null +++ b/misc/loop/.env.example @@ -0,0 +1,6 @@ +FAUCET_MNEMONIC= + +COUNTER_MNEMONIC= + +CAPTCHA_SITE_KEY= +CAPTCHA_SECRET_KEY= diff --git a/misc/loop/.gitignore b/misc/loop/.gitignore index 641b553abe6..bacc1a0c085 100644 --- a/misc/loop/.gitignore +++ b/misc/loop/.gitignore @@ -1,3 +1,4 @@ /portalloopd /backups /traefik/letsencrypt +.env diff --git a/misc/loop/Makefile b/misc/loop/Makefile index 3966cd42323..3ad86221ccd 100644 --- a/misc/loop/Makefile +++ b/misc/loop/Makefile @@ -1,18 +1,20 @@ -all: docker.start +PULL_GH_SCRIPT := ./scripts/pull-gh.sh + +all: docker.start.prod + +docker.start.prod: # Start the production portal loop + docker compose -f docker-compose.yml -f docker-compose.override.prod.yml up -d docker.start: # Start the portal loop docker compose up -d +docker.ci: # Start the portal loop for CI + docker compose up -d portalloopd traefik + docker.stop: # Stop the portal loop docker compose down docker rm -f $(docker ps -aq --filter "label=the-portal-loop") -docker.build: # (re)Build snapshotter image - docker compose build - -docker.pull: # Pull new images to update versions - docker compose pull - portalloopd.bash: # Get a bash command inside of the portalloopd container docker compose exec portalloopd bash @@ -20,3 +22,10 @@ switch: portalloopd.switch portalloopd.switch: # Force switch the portal loop with latest image docker compose exec portalloopd switch + +prepare-exports: + chmod +x $(PULL_GH_SCRIPT) && ./$(PULL_GH_SCRIPT) + +pull-exports: docker.stop prepare-exports + docker.start.prod + diff --git a/misc/loop/README.md b/misc/loop/README.md index ce02c83b67c..80618e1e5c7 100644 --- a/misc/loop/README.md +++ b/misc/loop/README.md @@ -1,4 +1,4 @@ -# The portal loop :infinity: +# The portal loop :infinity: ## What is it? @@ -6,17 +6,15 @@ It's a Gnoland node that aim to run with always the latest version of gno and ne For more information, see issue on github [gnolang/gno#1239](https://github.com/gnolang/gno/issues/1239) - ## How to use Start the loop with: ```sh -$ docker compose up -d +docker compose up -d # or using the Makefile - -$ make +make docker.start ``` The [`portalloopd`](./cmd/portalloopd) binary is starting inside of the docker container `portalloopd` @@ -24,7 +22,7 @@ The [`portalloopd`](./cmd/portalloopd) binary is starting inside of the docker c This script is doing: - Setup the current portal-loop in read only mode -- Pull the latest version of [ghcr.io/gnolang/gno]() +- Pull the latest version of [ghcr.io/gnolang/gno](ghcr.io/gnolang/gno) - Backup the txs using [gnolang/tx-archive](https://github.com/gnolang/tx-archive) - Start a new docker container with the backups files - Changing the proxy (traefik) to redirect to the new portal loop @@ -40,3 +38,24 @@ You can find a [Makefile](./Makefile) to help you interact with the portal loop ```bash make portalloopd.switch ``` + +### Running in production + +- Create an `.env` file adding all the entries from `.env.example` +- Setup the DNS names present in the `docker-compose.yml` file +- run using `make all` + +### Pulling in Portal Loop state `from tx-exports` + +To pull Portal Loop state from tx-exports, run the following make directive: + +```bash +make pull-exports +``` + +This will run the following steps: + +- stop any running portal loop containers -> Portal Loop will be down +- clone the `gnolang/tx-exports` repository and prepare the backup txs sheets located there as the genesis transactions + for Portal Loop +- start the portal loop containers -> Portal Loop will start back up again \ No newline at end of file diff --git a/misc/loop/backups/snapshots/.keep b/misc/loop/backups/snapshots/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/misc/loop/cmd/cmd_backup.go b/misc/loop/cmd/cmd_backup.go index f95e1e15a4a..2fa41c03ee4 100644 --- a/misc/loop/cmd/cmd_backup.go +++ b/misc/loop/cmd/cmd_backup.go @@ -12,8 +12,10 @@ import ( type backupCfg struct { rpcAddr string traefikGnoFile string - backupDir string hostPWD string + + masterBackupFile string + snapshotsDir string } func (c *backupCfg) RegisterFlags(fs *flag.FlagSet) { @@ -21,8 +23,8 @@ func (c *backupCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("HOST_PWD", os.Getenv("PWD")) } - if os.Getenv("BACKUP_DIR") == "" { - os.Setenv("BACKUP_DIR", "./backups") + if os.Getenv("SNAPSHOTS_DIR") == "" { + os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots") } if os.Getenv("RPC_URL") == "" { @@ -37,10 +39,15 @@ func (c *backupCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml") } + if os.Getenv("MASTER_BACKUP_FILE") == "" { + os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl") + } + fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url") fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file") - fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory") fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)") + fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path") + fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory") } func newBackupCmd(io commands.IO) *commands.Command { @@ -67,10 +74,11 @@ func execBackup(ctx context.Context, cfg *backupCfg) error { portalLoop := &snapshotter{} portalLoop, err = NewSnapshotter(dockerClient, config{ - backupDir: cfg.backupDir, - rpcAddr: cfg.rpcAddr, - hostPWD: cfg.hostPWD, - traefikGnoFile: cfg.traefikGnoFile, + snapshotsDir: cfg.snapshotsDir, + masterBackupFile: cfg.masterBackupFile, + rpcAddr: cfg.rpcAddr, + hostPWD: cfg.hostPWD, + traefikGnoFile: cfg.traefikGnoFile, }) if err != nil { return err diff --git a/misc/loop/cmd/cmd_serve.go b/misc/loop/cmd/cmd_serve.go index 61303041b34..f796f2268f6 100644 --- a/misc/loop/cmd/cmd_serve.go +++ b/misc/loop/cmd/cmd_serve.go @@ -16,8 +16,10 @@ import ( type serveCfg struct { rpcAddr string traefikGnoFile string - backupDir string hostPWD string + + masterBackupFile string + snapshotsDir string } func (c *serveCfg) RegisterFlags(fs *flag.FlagSet) { @@ -25,8 +27,8 @@ func (c *serveCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("HOST_PWD", os.Getenv("PWD")) } - if os.Getenv("BACKUP_DIR") == "" { - os.Setenv("BACKUP_DIR", "./backups") + if os.Getenv("SNAPSHOTS_DIR") == "" { + os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots") } if os.Getenv("RPC_URL") == "" { @@ -41,13 +43,18 @@ func (c *serveCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml") } + if os.Getenv("MASTER_BACKUP_FILE") == "" { + os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl") + } + fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url") fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file") - fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory") fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)") + fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path") + fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory") } -func newServeCmd(io commands.IO) *commands.Command { +func newServeCmd(_ commands.IO) *commands.Command { cfg := &serveCfg{} return commands.NewCommand( @@ -89,10 +96,11 @@ func execServe(ctx context.Context, cfg *serveCfg, args []string) error { // the loop for { portalLoop, err = NewSnapshotter(dockerClient, config{ - backupDir: cfg.backupDir, - rpcAddr: cfg.rpcAddr, - hostPWD: cfg.hostPWD, - traefikGnoFile: cfg.traefikGnoFile, + snapshotsDir: cfg.snapshotsDir, + masterBackupFile: cfg.masterBackupFile, + rpcAddr: cfg.rpcAddr, + hostPWD: cfg.hostPWD, + traefikGnoFile: cfg.traefikGnoFile, }) if err != nil { return err diff --git a/misc/loop/cmd/cmd_switch.go b/misc/loop/cmd/cmd_switch.go index 02f770cb61c..80487c2805d 100644 --- a/misc/loop/cmd/cmd_switch.go +++ b/misc/loop/cmd/cmd_switch.go @@ -12,8 +12,10 @@ import ( type switchCfg struct { rpcAddr string traefikGnoFile string - backupDir string hostPWD string + + masterBackupFile string + snapshotsDir string } func (c *switchCfg) RegisterFlags(fs *flag.FlagSet) { @@ -21,8 +23,8 @@ func (c *switchCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("HOST_PWD", os.Getenv("PWD")) } - if os.Getenv("BACKUP_DIR") == "" { - os.Setenv("BACKUP_DIR", "./backups") + if os.Getenv("SNAPSHOTS_DIR") == "" { + os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots") } if os.Getenv("RPC_URL") == "" { @@ -37,10 +39,15 @@ func (c *switchCfg) RegisterFlags(fs *flag.FlagSet) { os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml") } + if os.Getenv("MASTER_BACKUP_FILE") == "" { + os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl") + } + fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url") fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file") - fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory") fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)") + fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path") + fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory") } func newSwitchCmd(io commands.IO) *commands.Command { @@ -67,10 +74,11 @@ func execSwitch(ctx context.Context, cfg *switchCfg) error { portalLoop := &snapshotter{} portalLoop, err = NewSnapshotter(dockerClient, config{ - backupDir: cfg.backupDir, - rpcAddr: cfg.rpcAddr, - hostPWD: cfg.hostPWD, - traefikGnoFile: cfg.traefikGnoFile, + snapshotsDir: cfg.snapshotsDir, + masterBackupFile: cfg.masterBackupFile, + rpcAddr: cfg.rpcAddr, + hostPWD: cfg.hostPWD, + traefikGnoFile: cfg.traefikGnoFile, }) if err != nil { return err diff --git a/misc/loop/cmd/snapshotter.go b/misc/loop/cmd/snapshotter.go index 06562e2c5c5..2dda5d568d9 100644 --- a/misc/loop/cmd/snapshotter.go +++ b/misc/loop/cmd/snapshotter.go @@ -42,19 +42,22 @@ type snapshotter struct { type config struct { rpcAddr string traefikGnoFile string - backupDir string - hostPWD string + + snapshotsDir string + masterBackupFile string + + hostPWD string } func NewSnapshotter(dockerClient *client.Client, cfg config) (*snapshotter, error) { timenow := time.Now() now := fmt.Sprintf("%s_%v", timenow.Format("2006-01-02_"), timenow.UnixNano()) - backupFile, err := filepath.Abs(cfg.backupDir + "/backup.jsonl") + backupFile, err := filepath.Abs(cfg.masterBackupFile) if err != nil { return nil, err } - instanceBackupFile, err := filepath.Abs(fmt.Sprintf("%s/backup_%s.jsonl", cfg.backupDir, now)) + instanceBackupFile, err := filepath.Abs(fmt.Sprintf("%s/backup_%s.jsonl", cfg.snapshotsDir, now)) if err != nil { return nil, err } diff --git a/misc/loop/docker-compose.override.prod.yml b/misc/loop/docker-compose.override.prod.yml new file mode 100644 index 00000000000..cdc38dc3a73 --- /dev/null +++ b/misc/loop/docker-compose.override.prod.yml @@ -0,0 +1,6 @@ +services: + + portalloopd: + image: ghcr.io/gnolang/gno/portalloopd:latest + ports: + - 127.0.0.1:9090:9090 diff --git a/misc/loop/docker-compose.production.yml b/misc/loop/docker-compose.production.yml deleted file mode 100644 index 2afa013de64..00000000000 --- a/misc/loop/docker-compose.production.yml +++ /dev/null @@ -1,142 +0,0 @@ -version: "3" - -networks: - portal-loop: - name: portal-loop - driver: bridge - ipam: - config: - - subnet: 172.177.0.0/16 - -services: - traefik: - image: "traefik:v2.10" - restart: unless-stopped - command: - - "--api.insecure=true" - - "--providers.file=true" - - "--providers.file.watch=true" - - "--providers.file.directory=/etc/traefik/configs" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.rpc.address=:26657" - - "--entrypoints.web.http.redirections.entrypoint.to=websecure" - - "--entrypoints.web.http.redirections.entrypoint.scheme=https" - - "--entrypoints.web.http.redirections.entrypoint.permanent=true" - - "--entryPoints.web.forwardedHeaders.insecure" - - "--entrypoints.traefik.address=:8080" - - - "--entrypoints.websecure.address=:443" - # - "--certificatesresolvers.le.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" - - "--certificatesresolvers.le.acme.tlschallenge=true" - - "--certificatesresolvers.le.acme.email=dev@gno.land" - - "--certificatesresolvers.le.acme.storage=/letsencrypt/acme.json" - networks: - - portal-loop - ports: - - "80:80" - - "443:443" - - "26657:26657" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - ./traefik:/etc/traefik/configs - - ./traefik/letsencrypt:/letsencrypt - - gnoweb: - image: ghcr.io/gnolang/gno/gnoweb:master - restart: unless-stopped - env_file: ".env" - entrypoint: - - gnoweb - - --bind=0.0.0.0:8888 - - --remote=traefik:26657 - - --faucet-url=https://faucet-api.gno.land - - --captcha-site=$CAPTCHA_SITE_KEY - - --with-analytics - - --help-chainid=portal-loop - - --help-remote=https://rpc.gno.land:443 - networks: - - portal-loop - labels: - com.centurylinklabs.watchtower.enable: "true" - traefik.enable: "true" - traefik.http.routers.gnoweb.entrypoints: "web,websecure" - traefik.http.routers.gnoweb.rule: "Host(`gno.land`) || Host(`www.gno.land`)" - traefik.http.routers.gnoweb.tls: "true" - traefik.http.routers.gnoweb.tls.certresolver: "le" - - gnofaucet: - image: ghcr.io/gnolang/gno/gnofaucet-slim - networks: - - portal-loop - command: - - "serve" - - "--listen-address=0.0.0.0:5050" - - "--chain-id=portal-loop" - - "--is-behind-proxy=true" - - "--mnemonic=${FAUCET_MNEMONIC}" - - "--num-accounts=1" - - "--remote=http://traefik:26657" - - "--captcha-secret=${CAPTCHA_SECRET_KEY}" - env_file: ".env" - # environment: - # from .env - # - RECAPTCHA_SECRET_KEY - labels: - com.centurylinklabs.watchtower.enable: "true" - traefik.enable: "true" - traefik.http.routers.gnofaucet-api.entrypoints: "websecure" - traefik.http.routers.gnofaucet-api.rule: "Host(`faucet-api.gno.land`)" - traefik.http.routers.gnofaucet-api.tls: "true" - traefik.http.routers.gnofaucet-api.tls.certresolver: "le" - traefik.http.middlewares.gnofaucet-ratelimit.ratelimit.average: "6" - traefik.http.middlewares.gnofaucet-ratelimit.ratelimit.period: "1m" - - portalloopd: - image: ghcr.io/gnolang/gno/portalloopd - restart: unless-stopped - volumes: - - ./scripts:/scripts - - ./backups:/backups - - ./traefik:/etc/traefik/configs - - "/var/run/docker.sock:/var/run/docker.sock:ro" - networks: - - portal-loop - ports: - - 127.0.0.1:9090:9090 - environment: - HOST_PWD: $PWD - BACKUP_DIR: "/backups" - RPC_URL: "http://traefik:26657" - PROM_ADDR: "0.0.0.0:9090" - TRAEFIK_GNO_FILE: "/etc/traefik/configs/gno.yml" - extra_hosts: - - host.docker.internal:host-gateway - labels: - - "com.centurylinklabs.watchtower.enable=true" - - autocounterd: - image: ghcr.io/gnolang/gno/autocounterd - restart: unless-stopped - env_file: ".env" - command: - - "start" - - "--chain-id=portal-loop" - - "--interval=15m" - - "--mnemonic=${COUNTER_MNEMONIC}" - - "--rpc=http://traefik:26657" - networks: - - portal-loop - labels: - com.centurylinklabs.watchtower.enable: "true" - - watchtower: - image: containrrr/watchtower - command: --interval 30 --http-api-metrics --label-enable - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - WATCHTOWER_HTTP_API_TOKEN: "mytoken" - ports: - - 127.0.0.1:8000:8080 diff --git a/misc/loop/docker-compose.yml b/misc/loop/docker-compose.yml index ed2fe7192f5..c3adc6a39ea 100644 --- a/misc/loop/docker-compose.yml +++ b/misc/loop/docker-compose.yml @@ -1,11 +1,8 @@ -version: "3" - networks: portal-loop: name: portal-loop driver: bridge ipam: - driver: default config: - subnet: 172.42.0.0/16 @@ -18,53 +15,92 @@ services: - "--providers.file=true" - "--providers.file.watch=true" - "--providers.file.directory=/etc/traefik/configs" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.rpc.address=:26657" + - "--entrypoints.websecure.address=:443" - "--entrypoints.web.address=:80" - - "--entrypoints.private.address=:26657" - - "--entrypoints.traefik.address=:8080" + - "--entrypoints.web.http.redirections.entrypoint.to=websecure" + - "--entrypoints.web.http.redirections.entrypoint.scheme=https" + - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + - "--entryPoints.web.forwardedHeaders.insecure" + - "--certificatesresolvers.le.acme.tlschallenge=true" + - "--certificatesresolvers.le.acme.email=dev@gno.land" networks: - portal-loop ports: - "80:80" - - "8080:8080" + - "443:443" - "26657:26657" volumes: - ./traefik:/etc/traefik/configs + - "/var/run/docker.sock:/var/run/docker.sock:ro" gnoweb: image: ghcr.io/gnolang/gno/gnoweb:master restart: unless-stopped - networks: - - portal-loop - ports: - - 8888:8888 + env_file: ".env" entrypoint: - gnoweb - --bind=0.0.0.0:8888 - --remote=traefik:26657 - - --faucet-url - - "http://localhost:5050" - - --help-chainid + - --faucet-url=https://faucet-api.gno.land + - --captcha-site=$CAPTCHA_SITE_KEY + - --with-analytics + - --help-chainid=portal-loop + - --help-remote=https://rpc.gno.land:443 + networks: - portal-loop - - --help-remote - - http://127.0.0.1:26657 + labels: + com.centurylinklabs.watchtower.enable: "true" + traefik.enable: "true" + traefik.http.routers.gnoweb.entrypoints: "web,websecure" + traefik.http.routers.gnoweb.rule: "Host(`gno.land`) || Host(`www.gno.land`)" + traefik.http.routers.gnoweb.tls: "true" + traefik.http.routers.gnoweb.tls.certresolver: "le" gnofaucet: - image: ghcr.io/gnolang/gno/gnofaucet-slim + image: ghcr.io/gnolang/gno/gnofaucet:master networks: - portal-loop - ports: - - 5050:5050 command: - "serve" - "--listen-address=0.0.0.0:5050" - "--chain-id=portal-loop" - # - "--is-behind-proxy=true" - - "--mnemonic=${MNEMONIC}" - # - "--num-accounts=1" + - "--is-behind-proxy=true" + - "--mnemonic=${FAUCET_MNEMONIC}" + - "--num-accounts=1" - "--remote=http://traefik:26657" - environment: - # from .env - - RECAPTCHA_SECRET_KEY + - "--captcha-secret=${CAPTCHA_SECRET_KEY}" + env_file: ".env" + labels: + com.centurylinklabs.watchtower.enable: "true" + traefik.enable: "true" + traefik.http.routers.gnofaucet-api.entrypoints: "websecure" + traefik.http.routers.gnofaucet-api.rule: "Host(`faucet-api.gno.land`)" + traefik.http.routers.gnofaucet-api.tls: "true" + traefik.http.routers.gnofaucet-api.tls.certresolver: "le" + traefik.http.middlewares.gnofaucet-ratelimit.ratelimit.average: "6" + traefik.http.middlewares.gnofaucet-ratelimit.ratelimit.period: "1m" + + tx-indexer: + image: ghcr.io/gnolang/tx-indexer:latest + networks: + - portal-loop + entrypoint: + - /tx-indexer + - start + - "-http-rate-limit=500" + - "-listen-address=0.0.0.0:8546" + - "-max-slots=2000" + - "-remote=http://traefik:26657" + labels: + traefik.enable: "true" + traefik.http.routers.tx-indexer.entrypoints: "websecure" + traefik.http.routers.tx-indexer.rule: "Host(`indexer.portal.gnoteam.com`)" + traefik.http.routers.tx-indexer.tls: "true" + traefik.http.routers.tx-indexer.tls.certresolver: "le" + traefik.http.services.tx-indexer.loadbalancer.server.port: 8546 portalloopd: build: @@ -82,9 +118,38 @@ services: - 9090:9090 environment: HOST_PWD: $PWD - BACKUP_DIR: "/backups" + SNAPSHOTS_DIR: "/backups/snapshots" + MASTER_BACKUP_FILE: "/backups/backup.jsonl" RPC_URL: "http://traefik:26657" PROM_ADDR: "0.0.0.0:9090" TRAEFIK_GNO_FILE: "/etc/traefik/configs/gno.yml" extra_hosts: - host.docker.internal:host-gateway + labels: + - "com.centurylinklabs.watchtower.enable=true" + + autocounterd: + image: ghcr.io/gnolang/gno/autocounterd:latest + restart: unless-stopped + env_file: ".env" + command: + - "start" + - "--chain-id=portal-loop" + - "--interval=15m" + - "--mnemonic=${COUNTER_MNEMONIC}" + - "--rpc=http://traefik:26657" + networks: + - portal-loop + labels: + com.centurylinklabs.watchtower.enable: "true" + + watchtower: + image: containrrr/watchtower + command: --interval 30 --http-api-metrics --label-enable + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /home/devops/.docker/config.json:/config.json + environment: + WATCHTOWER_HTTP_API_TOKEN: "mytoken" + ports: + - 127.0.0.1:8000:8080 diff --git a/misc/loop/go.mod b/misc/loop/go.mod index 9fc5bfb2d57..f1c09cd9f82 100644 --- a/misc/loop/go.mod +++ b/misc/loop/go.mod @@ -65,8 +65,6 @@ require ( go.opentelemetry.io/otel/trace v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - go.uber.org/zap/exp v0.2.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.20.0 // indirect diff --git a/misc/loop/go.sum b/misc/loop/go.sum index 27ed94fecae..740cc629a21 100644 --- a/misc/loop/go.sum +++ b/misc/loop/go.sum @@ -201,14 +201,8 @@ go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt3 go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= -go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/misc/loop/scripts/pull-gh.sh b/misc/loop/scripts/pull-gh.sh new file mode 100755 index 00000000000..efbb360d551 --- /dev/null +++ b/misc/loop/scripts/pull-gh.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env sh + +TMP_DIR=temp-tx-exports + +# The master backup file will contain the ultimate txs backup +# that the portal loop use when looping (generating the genesis) +MASTER_BACKUP_FILE="backup.jsonl" + +# Clones the portal loop backups subdirectory, located in BACKUPS_REPO (tx-exports) +pullGHBackups () { + BACKUPS_REPO=https://github.com/gnolang/tx-exports.git + BACKUPS_REPO_PATH="portal-loop" + + # Clone just the root folder of the same name + git clone --depth 1 --no-checkout $BACKUPS_REPO + cd "$(basename "$BACKUPS_REPO" .git)" || exit 1 + + # Clone just the backups path in the cloned repo + git sparse-checkout set $BACKUPS_REPO_PATH + git checkout + + # Go back to the parent directory + cd .. +} + +# Create the temporary working dir +rm -rf $TMP_DIR && mkdir $TMP_DIR +cd $TMP_DIR || exit 1 + +# Pull the backup repo data +pullGHBackups + +# Combine the pulled backups into a single backup file +TXS_BACKUPS_PREFIX="backup_portal_loop_txs_" + +find . -type f -name "${TXS_BACKUPS_PREFIX}*.jsonl" | sort | xargs cat > "temp_$MASTER_BACKUP_FILE" + +BACKUPS_DIR="../backups" +TIMESTAMP=$(date +%s) + +# Check if the master backup file already exists +if [ -e "$BACKUPS_DIR/$MASTER_BACKUP_FILE" ]; then + # Back up the existing master txs file + echo "Master backup file exists, backing up..." + mv "$BACKUPS_DIR/$MASTER_BACKUP_FILE" "$BACKUPS_DIR/${MASTER_BACKUP_FILE}-legacy-$TIMESTAMP" + + echo "Renamed $MASTER_BACKUP_FILE to ${MASTER_BACKUP_FILE}-legacy-$TIMESTAMP" +fi + +# Use the GitHub state as the canonical backup +mv "temp_$MASTER_BACKUP_FILE" "$BACKUPS_DIR/$MASTER_BACKUP_FILE" +echo "Moved temp_$MASTER_BACKUP_FILE to $BACKUPS_DIR/$MASTER_BACKUP_FILE" + +# Clean up the temporary directory +cd .. +rm -rf $TMP_DIR diff --git a/misc/loop/scripts/start.sh b/misc/loop/scripts/start.sh index 76869ccb4bd..6dd57b2c041 100755 --- a/misc/loop/scripts/start.sh +++ b/misc/loop/scripts/start.sh @@ -12,7 +12,7 @@ SEEDS=${SEEDS:-""} PERSISTENT_PEERS=${PERSISTENT_PEERS:-""} echo "" >> /gnoroot/gno.land/genesis/genesis_txs.jsonl -cat ${GENESIS_BACKUP_FILE} >> /gnoroot/gno.land/genesis/genesis_txs.jsonl +cat "${GENESIS_BACKUP_FILE}" >> /gnoroot/gno.land/genesis/genesis_txs.jsonl # Initialize the secrets gnoland secrets init diff --git a/misc/loop/tools.go b/misc/loop/tools.go deleted file mode 100644 index 789ea2949f0..00000000000 --- a/misc/loop/tools.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build tools - -package tools - -import ( - _ "github.com/gnolang/gno/gno.land/cmd/gnoland" - _ "github.com/gnolang/tx-archive/cmd" -) diff --git a/misc/loop/traefik/gno.yml b/misc/loop/traefik/gno.yml index 7e65930889d..09d7e0d0815 100644 --- a/misc/loop/traefik/gno.yml +++ b/misc/loop/traefik/gno.yml @@ -11,7 +11,7 @@ http: gno-portal-loop-local: service: gno-portal-loop rule: "PathPrefix(`/`)" - entrypoints: ["private"] + entrypoints: [ "rpc" ] middlewares: [] gno-portal-loop: @@ -19,11 +19,11 @@ http: tls: certResolver: le rule: "Host(`rpc.gno.land`) || Host(`rpc.portal.gnoteam.com`)" - entrypoints: ["web", "websecure"] + entrypoints: [ "web", "websecure" ] middlewares: [] services: gno-portal-loop: loadBalancer: servers: - - url: "http://172.42.0.2:26657" + - url: "http://172.42.0.4:26657" diff --git a/misc/loop/traefik/gnofaucet.yml b/misc/loop/traefik/gnofaucet.yml deleted file mode 100644 index 25dd092a504..00000000000 --- a/misc/loop/traefik/gnofaucet.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -http: - routers: - gnofaucet-local: - service: gnofaucet - rule: "Host(`faucet.portal.gno.local`)" - entrypoints: ["web", "websecure", "private"] - middlewares: [] - - gnofaucet: - service: gnofaucet - rule: "Host(`faucet.gno.land`) || Host(`faucet.portal.gnoteam.com`)" - tls: - certResolver: le - entrypoints: ["web", "websecure"] - middlewares: [] - - services: - gnofaucet: - loadBalancer: - servers: - - url: "http://localhost:9000" diff --git a/misc/loop/traefik/gnoweb.yml b/misc/loop/traefik/gnoweb.yml deleted file mode 100644 index 8bce0f4bfb6..00000000000 --- a/misc/loop/traefik/gnoweb.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -http: - routers: - gnoweb-local: - service: gnoweb - rule: "Host(`portal.gno.local`)" - entrypoints: ["web", "websecure", "private"] - middlewares: [] - - gnoweb: - service: gnoweb - rule: "Host(`gno.land`) || Host(`portal.gnoteam.com`)" - tls: - certResolver: le - entrypoints: ["web", "websecure"] - middlewares: [] - - services: - gnoweb: - loadBalancer: - servers: - - url: "http://localhost:8888"