chore: sync portal loop machine and gnolang/gno repo (#3173)

## Description

This PR updates the outdated repo portal loop Dockerfile to align with
the Dockerfile on the actual machine, that is more up to date.

It also allows the portal loop to be able to pull and be loaded from
state on `tx-exports`, using `make pull-exports`.

<details><summary>Contributors' checklist...</summary>

- [x] Added new tests, or not needed, or not feasible
- [x] Provided an example (e.g. screenshot) to aid review or the PR is
self-explanatory
- [x] Updated the official documentation or not needed
- [x] No breaking changes were made, or a `BREAKING CHANGE: xxx` message
was included in the description
- [ ] Added references to related issues and PRs
- [ ] Provided any useful hints for running manual tests
</details>

---------

Co-authored-by: Sergio Maria Matone <[email protected]>

.github/workflows/portal-loop.yml

@@ -57,13 +57,12 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v4
 
-      - name: "Setup the images"
+      - name: "Setup The portal loop docker compose"
         run: |
           cd misc/loop
-
-          docker compose build
-          docker compose pull
-          docker compose up -d
+          echo "Making docker compose happy"
+          touch .env
+          make docker.ci
 
       - name: "Test1 - Portal loop start gnoland"
         run: |

misc/loop/.env.example

@@ -0,0 +1,6 @@
+FAUCET_MNEMONIC=
+
+COUNTER_MNEMONIC=
+
+CAPTCHA_SITE_KEY=
+CAPTCHA_SECRET_KEY=

misc/loop/.gitignore

@@ -1,3 +1,4 @@
 /portalloopd
 /backups
 /traefik/letsencrypt
+.env

misc/loop/Makefile

@@ -1,22 +1,31 @@
-all: docker.start
+PULL_GH_SCRIPT := ./scripts/pull-gh.sh
+
+all: docker.start.prod
+
+docker.start.prod: # Start the production portal loop
+	docker compose -f docker-compose.yml -f docker-compose.override.prod.yml up -d
 
 docker.start: # Start the portal loop
 	docker compose up -d
 
+docker.ci: # Start the portal loop for CI
+	docker compose up -d portalloopd traefik
+
 docker.stop: # Stop the portal loop
 	docker compose down
 	docker rm -f $(docker ps -aq --filter "label=the-portal-loop")
 
-docker.build: # (re)Build snapshotter image
-	docker compose build
-
-docker.pull: # Pull new images to update versions
-	docker compose pull
-
 portalloopd.bash: # Get a bash command inside of the portalloopd container
 	docker compose exec portalloopd bash
 
 switch: portalloopd.switch
 
 portalloopd.switch: # Force switch the portal loop with latest image
 	docker compose exec portalloopd switch
+
+prepare-exports:
+	chmod +x $(PULL_GH_SCRIPT) && ./$(PULL_GH_SCRIPT)
+
+pull-exports: docker.stop prepare-exports
+	docker.start.prod
+

misc/loop/README.md

@@ -1,30 +1,28 @@
-# The portal loop :infinity: 
+# The portal loop :infinity:
 
 ## What is it?
 
 It's a Gnoland node that aim to run with always the latest version of gno and never loose transactions history.
 
 For more information, see issue on github [gnolang/gno#1239](https://github.com/gnolang/gno/issues/1239)
 
-
 ## How to use
 
 Start the loop with:
 
 ```sh
-$ docker compose up -d
+docker compose up -d
 
 # or using the Makefile
-
-$ make
+make docker.start
 ```
 
 The [`portalloopd`](./cmd/portalloopd) binary is starting inside of the docker container `portalloopd`
 
 This script is doing:
 
 - Setup the current portal-loop in read only mode
-- Pull the latest version of [ghcr.io/gnolang/gno]()
+- Pull the latest version of [ghcr.io/gnolang/gno](ghcr.io/gnolang/gno)
 - Backup the txs using [gnolang/tx-archive](https://github.com/gnolang/tx-archive)
 - Start a new docker container with the backups files
 - Changing the proxy (traefik) to redirect to the new portal loop
@@ -40,3 +38,24 @@ You can find a [Makefile](./Makefile) to help you interact with the portal loop
 ```bash
 make portalloopd.switch
 ```
+
+### Running in production
+
+- Create an `.env` file adding all the entries from `.env.example`
+- Setup the DNS names present in the `docker-compose.yml` file
+- run using `make all`
+
+### Pulling in Portal Loop state `from tx-exports`
+
+To pull Portal Loop state from tx-exports, run the following make directive:
+
+```bash
+make pull-exports
+```
+
+This will run the following steps:
+
+- stop any running portal loop containers -> Portal Loop will be down
+- clone the `gnolang/tx-exports` repository and prepare the backup txs sheets located there as the genesis transactions
+  for Portal Loop
+- start the portal loop containers -> Portal Loop will start back up again

misc/loop/cmd/cmd_backup.go

@@ -12,17 +12,19 @@ import (
 type backupCfg struct {
 	rpcAddr        string
 	traefikGnoFile string
-	backupDir      string
 	hostPWD        string
+
+	masterBackupFile string
+	snapshotsDir     string
 }
 
 func (c *backupCfg) RegisterFlags(fs *flag.FlagSet) {
 	if os.Getenv("HOST_PWD") == "" {
 		os.Setenv("HOST_PWD", os.Getenv("PWD"))
 	}
 
-	if os.Getenv("BACKUP_DIR") == "" {
-		os.Setenv("BACKUP_DIR", "./backups")
+	if os.Getenv("SNAPSHOTS_DIR") == "" {
+		os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots")
 	}
 
 	if os.Getenv("RPC_URL") == "" {
@@ -37,10 +39,15 @@ func (c *backupCfg) RegisterFlags(fs *flag.FlagSet) {
 		os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml")
 	}
 
+	if os.Getenv("MASTER_BACKUP_FILE") == "" {
+		os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl")
+	}
+
 	fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url")
 	fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file")
-	fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory")
 	fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)")
+	fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path")
+	fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory")
 }
 
 func newBackupCmd(io commands.IO) *commands.Command {
@@ -67,10 +74,11 @@ func execBackup(ctx context.Context, cfg *backupCfg) error {
 	portalLoop := &snapshotter{}
 
 	portalLoop, err = NewSnapshotter(dockerClient, config{
-		backupDir:      cfg.backupDir,
-		rpcAddr:        cfg.rpcAddr,
-		hostPWD:        cfg.hostPWD,
-		traefikGnoFile: cfg.traefikGnoFile,
+		snapshotsDir:     cfg.snapshotsDir,
+		masterBackupFile: cfg.masterBackupFile,
+		rpcAddr:          cfg.rpcAddr,
+		hostPWD:          cfg.hostPWD,
+		traefikGnoFile:   cfg.traefikGnoFile,
 	})
 	if err != nil {
 		return err

misc/loop/cmd/cmd_serve.go

@@ -16,17 +16,19 @@ import (
 type serveCfg struct {
 	rpcAddr        string
 	traefikGnoFile string
-	backupDir      string
 	hostPWD        string
+
+	masterBackupFile string
+	snapshotsDir     string
 }
 
 func (c *serveCfg) RegisterFlags(fs *flag.FlagSet) {
 	if os.Getenv("HOST_PWD") == "" {
 		os.Setenv("HOST_PWD", os.Getenv("PWD"))
 	}
 
-	if os.Getenv("BACKUP_DIR") == "" {
-		os.Setenv("BACKUP_DIR", "./backups")
+	if os.Getenv("SNAPSHOTS_DIR") == "" {
+		os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots")
 	}
 
 	if os.Getenv("RPC_URL") == "" {
@@ -41,13 +43,18 @@ func (c *serveCfg) RegisterFlags(fs *flag.FlagSet) {
 		os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml")
 	}
 
+	if os.Getenv("MASTER_BACKUP_FILE") == "" {
+		os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl")
+	}
+
 	fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url")
 	fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file")
-	fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory")
 	fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)")
+	fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path")
+	fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory")
 }
 
-func newServeCmd(io commands.IO) *commands.Command {
+func newServeCmd(_ commands.IO) *commands.Command {
 	cfg := &serveCfg{}
 
 	return commands.NewCommand(
@@ -89,10 +96,11 @@ func execServe(ctx context.Context, cfg *serveCfg, args []string) error {
 	// the loop
 	for {
 		portalLoop, err = NewSnapshotter(dockerClient, config{
-			backupDir:      cfg.backupDir,
-			rpcAddr:        cfg.rpcAddr,
-			hostPWD:        cfg.hostPWD,
-			traefikGnoFile: cfg.traefikGnoFile,
+			snapshotsDir:     cfg.snapshotsDir,
+			masterBackupFile: cfg.masterBackupFile,
+			rpcAddr:          cfg.rpcAddr,
+			hostPWD:          cfg.hostPWD,
+			traefikGnoFile:   cfg.traefikGnoFile,
 		})
 		if err != nil {
 			return err

misc/loop/cmd/cmd_switch.go

@@ -12,17 +12,19 @@ import (
 type switchCfg struct {
 	rpcAddr        string
 	traefikGnoFile string
-	backupDir      string
 	hostPWD        string
+
+	masterBackupFile string
+	snapshotsDir     string
 }
 
 func (c *switchCfg) RegisterFlags(fs *flag.FlagSet) {
 	if os.Getenv("HOST_PWD") == "" {
 		os.Setenv("HOST_PWD", os.Getenv("PWD"))
 	}
 
-	if os.Getenv("BACKUP_DIR") == "" {
-		os.Setenv("BACKUP_DIR", "./backups")
+	if os.Getenv("SNAPSHOTS_DIR") == "" {
+		os.Setenv("SNAPSHOTS_DIR", "./backups/snapshots")
 	}
 
 	if os.Getenv("RPC_URL") == "" {
@@ -37,10 +39,15 @@ func (c *switchCfg) RegisterFlags(fs *flag.FlagSet) {
 		os.Setenv("TRAEFIK_GNO_FILE", "./traefik/gno.yml")
 	}
 
+	if os.Getenv("MASTER_BACKUP_FILE") == "" {
+		os.Setenv("MASTER_BACKUP_FILE", "./backups/backup.jsonl")
+	}
+
 	fs.StringVar(&c.rpcAddr, "rpc", os.Getenv("RPC_URL"), "tendermint rpc url")
 	fs.StringVar(&c.traefikGnoFile, "traefik-gno-file", os.Getenv("TRAEFIK_GNO_FILE"), "traefik gno file")
-	fs.StringVar(&c.backupDir, "backup-dir", os.Getenv("BACKUP_DIR"), "backup directory")
 	fs.StringVar(&c.hostPWD, "pwd", os.Getenv("HOST_PWD"), "host pwd (for docker usage)")
+	fs.StringVar(&c.masterBackupFile, "master-backup-file", os.Getenv("MASTER_BACKUP_FILE"), "master txs backup file path")
+	fs.StringVar(&c.snapshotsDir, "snapshots-dir", os.Getenv("SNAPSHOTS_DIR"), "snapshots directory")
 }
 
 func newSwitchCmd(io commands.IO) *commands.Command {
@@ -67,10 +74,11 @@ func execSwitch(ctx context.Context, cfg *switchCfg) error {
 	portalLoop := &snapshotter{}
 
 	portalLoop, err = NewSnapshotter(dockerClient, config{
-		backupDir:      cfg.backupDir,
-		rpcAddr:        cfg.rpcAddr,
-		hostPWD:        cfg.hostPWD,
-		traefikGnoFile: cfg.traefikGnoFile,
+		snapshotsDir:     cfg.snapshotsDir,
+		masterBackupFile: cfg.masterBackupFile,
+		rpcAddr:          cfg.rpcAddr,
+		hostPWD:          cfg.hostPWD,
+		traefikGnoFile:   cfg.traefikGnoFile,
 	})
 	if err != nil {
 		return err

misc/loop/cmd/snapshotter.go

@@ -42,19 +42,22 @@ type snapshotter struct {
 type config struct {
 	rpcAddr        string
 	traefikGnoFile string
-	backupDir      string
-	hostPWD        string
+
+	snapshotsDir     string
+	masterBackupFile string
+
+	hostPWD string
 }
 
 func NewSnapshotter(dockerClient *client.Client, cfg config) (*snapshotter, error) {
 	timenow := time.Now()
 	now := fmt.Sprintf("%s_%v", timenow.Format("2006-01-02_"), timenow.UnixNano())
 
-	backupFile, err := filepath.Abs(cfg.backupDir + "/backup.jsonl")
+	backupFile, err := filepath.Abs(cfg.masterBackupFile)
 	if err != nil {
 		return nil, err
 	}
-	instanceBackupFile, err := filepath.Abs(fmt.Sprintf("%s/backup_%s.jsonl", cfg.backupDir, now))
+	instanceBackupFile, err := filepath.Abs(fmt.Sprintf("%s/backup_%s.jsonl", cfg.snapshotsDir, now))
 	if err != nil {
 		return nil, err
 	}

misc/loop/docker-compose.override.prod.yml

@@ -0,0 +1,6 @@
+services:
+
+  portalloopd:
+    image: ghcr.io/gnolang/gno/portalloopd:latest
+    ports:
+      - 127.0.0.1:9090:9090