single browser; graceful shutdown; deploy img

.github/workflows/deploy.yml

@@ -0,0 +1,68 @@
+name: deployment build
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout current repo
+        uses: actions/checkout@v3
+      - name: Checkout main gcsim repo
+        uses: actions/checkout@v2
+        with:
+          repository: genshinsim/gcsim
+          ref: main
+          path: ./gcsim
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.3"
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '20.5.1'
+          cache: yarn
+          cache-dependency-path: "**/yarn.lock"
+      - name: Install
+        working-directory: ./gcsim/ui
+        shell: bash
+        run: yarn install --immutable
+      - name: Build embed ui
+        working-directory: ./gcsim/ui
+        shell: bash
+        run: yarn workspace @gcsim/embed build
+      - name: List UI dist
+        working-directory: ./gcsim/ui/packages/embed/dist
+        shell: bash
+        run: |
+          pwd
+          ls -lh
+      - name: move dist into current repo folder
+        run: |
+          pwd
+          mkdir ./cmd/preview/dist
+          mv ./gcsim/ui/packages/embed/dist/* ./cmd/preview/dist
+          ls -lh ./cmd/preview
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ghcr.io/${{ github.repository }}
+      - name: Build go executable
+        working-directory: ./cmd/preview
+        run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build .
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: ./cmd/preview/
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

chrome.go

@@ -25,15 +25,15 @@ func (s *Server) listen() {
 	//only one unit of work at a time
 	var queue []string
 	done := make(chan resp)
-	count := 0
+	busy := false
 
 	for {
 		select {
 		case w := <-s.work:
 			queue = append(queue, w)
 			s.logger.Info("got work", "id", w)
 		case res := <-done:
-			count--
+			busy = false
 			if res.err != nil {
 				status := s.rdb.Set(context.Background(), res.id, res.err.Error(), 5*time.Second)
 				pubstatus := s.rdb.Publish(context.Background(), res.id, res.err.Error())
@@ -45,8 +45,8 @@ func (s *Server) listen() {
 			}
 		}
 		//TODO: more than 1 worker?
-		if count < 1 && len(queue) > 0 {
-			count++
+		if !busy && len(queue) > 0 {
+			busy = true
 			next := queue[0]
 			go s.doWork(next, done)
 			queue = queue[1:]
@@ -66,13 +66,7 @@ func (s *Server) doWork(id string, done chan resp) {
 }
 
 func (s *Server) generateSnapshot(url string) ([]byte, error) {
-	browser := rod.New().Client(s.l.MustClient())
-	err := browser.Connect()
-	if err != nil {
-		return nil, fmt.Errorf("error connecting to browser: %w", err)
-	}
-	log.Println("browser connect ok")
-	page, err := browser.Page(proto.TargetCreateTarget{})
+	page, err := s.browser.Page(proto.TargetCreateTarget{})
 	if err != nil {
 		return nil, fmt.Errorf("error creating page: %w", err)
 	}
@@ -94,13 +88,22 @@ func (s *Server) generateSnapshot(url string) ([]byte, error) {
 	_, err = page.Race().ElementFunc(func(p *rod.Page) (*rod.Element, error) {
 		res, err := page.Evaluate(rod.Eval(`(s, n) => document.querySelectorAll(s).length > n`, "div", 10))
 		if err != nil {
+			s.logger.Info("error querying for all divs", "err", err)
+			return nil, &rod.ElementNotFoundError{}
+		}
+		if !res.Value.Bool() {
+			return nil, &rod.ElementNotFoundError{}
+		}
+		res, err = page.Evaluate(rod.Eval(`(s) => document.querySelectorAll(s).length > 0`, "#images_loaded"))
+		if err != nil {
+			s.logger.Info("error querying for #images_loaded", "err", err)
 			return nil, &rod.ElementNotFoundError{}
 		}
-		if res.Value.Bool() {
-			return &rod.Element{}, nil
+		if !res.Value.Bool() {
+			return nil, &rod.ElementNotFoundError{}
 		}
 
-		return nil, &rod.ElementNotFoundError{}
+		return &rod.Element{}, nil
 	}).Element("#has-error").Handle(func(e *rod.Element) error {
 		str, err := e.Attribute("value")
 		// can't do much aobut this err here other than log it

cmd/preview/.gitignore

@@ -1 +1,3 @@
-dist
+dist
+preview
+.env

cmd/preview/docker-compose.yml

@@ -1,6 +1,15 @@
-version: "3.8"
+# version: "3.8"
 
 services:
+  cloudflared:
+    container_name: cloudflared
+    image: cloudflare/cloudflared:latest
+    command: "tunnel --no-autoupdate run --token $CF_TUNNEL_TOKEN"
+    volumes:
+      - ./cloudflared:/etc/cloudflared
+    environment:
+      - CF_TUNNEL_TOKEN
+    restart: unless-stopped
   redis:
     # name: redis-stack
     image: redis/redis-stack:latest
@@ -18,15 +27,16 @@ services:
       - rod
     build:
       context: .
-    volumes:
-      - /home/srliao/code/assets/assets:/assets
+    # volumes:
+    #   - /home/srliao/code/assets/assets:/assets
     environment:
       - PORT=7777
       - LAUNCHER_URL=ws://rod:7317
-      - AUTH_KEY=SOME_TEST_KEY_NOT_FOR_PROD
-      - ASSETS_DATA_PATH=/assets
+      - ASSETS_PATH=/assets
       - REDIS_URL=redis:6379
       - PREVIEW_URL=http://preview:7777
+      - PROXY_TO=https://gcsim.app
+      - AUTH_KEY
     ports:
       - 7777:7777
     restart: unless-stopped

cmd/preview/main.go

@@ -1,11 +1,18 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"embed"
+	"errors"
 	"fmt"
 	"log"
 	"math/big"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"github.com/caarlos0/env/v10"
 	"github.com/genshinsim/preview"
@@ -55,7 +62,7 @@ func main() {
 	server, err := preview.New(content, redis.UniversalOptions{
 		Addrs: cfg.RedisURL,
 		DB:    cfg.RedisDB,
-	}, cfg.Host+":"+cfg.Port, cfg.LauncherURL, cfg.PreviewURL, cfg.AuthKey)
+	}, cfg.LauncherURL, cfg.PreviewURL, cfg.AuthKey)
 
 	panicErr(err)
 
@@ -77,8 +84,41 @@ func main() {
 		panicErr(server.SetOpts(preview.WithCacheTTL(cfg.CacheTTLInSec)))
 	}
 
-	log.Println("starting img generation listener")
-	log.Fatal(server.Start())
+	err = server.Init()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	httpServer := &http.Server{
+		Addr:    cfg.Host + ":" + cfg.Port,
+		Handler: server,
+	}
+
+	go func() {
+		if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("HTTP server error: %v", err)
+		}
+		log.Println("Stopped serving new connections.")
+	}()
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	<-sigChan
+
+	shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 10*time.Second)
+	defer shutdownRelease()
+
+	if err := httpServer.Shutdown(shutdownCtx); err != nil {
+		log.Println("HTTP graceful shutdown encountered error, forcing shutdown")
+		//force shut down
+		err := httpServer.Close()
+		log.Println("Force shut down completed with error: ", err)
+		log.Println("Shutting down browsers: ", server.Shutdown())
+		//shut down browsers
+		log.Fatalf("HTTP shutdown error: %v", err)
+	}
+	log.Println("Graceful shutdown complete.")
+	log.Println("Shutting down browsers: ", server.Shutdown())
 }
 
 func panicErr(err error) {

handler.go

@@ -18,7 +18,7 @@ import (
 
 func (s *Server) handleProxy(prefix string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		s.logger.Info("proxying request", "path", prefix, "url", r.URL)
+		s.logger.Info("proxying request", "prefix", prefix, "url", r.URL)
 		r.Host = s.proxyTarget.Host
 		s.proxy.ServeHTTP(w, r)
 	}
@@ -90,7 +90,7 @@ func (s *Server) handleImageRequest(src string) http.HandlerFunc {
 		defer pubsub.Close()
 
 		res := s.rdb.Get(ctx, id)
-		s.logger.Info("got get from redis", "res", res)
+		s.logger.Info("got get from redis", "res_length", len(res.Val()))
 		switch res.Err() {
 		case nil:
 			if val := res.Val(); !strings.HasPrefix(val, "wip") {

server.go

@@ -15,19 +15,20 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
+	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/redis/go-redis/v9"
 )
 
 type serverCfg func(s *Server) error
 
 type Server struct {
+	*chi.Mux
+
 	logger     *slog.Logger
-	router     *chi.Mux
 	rdb        redis.UniversalClient
 	work       chan string
 	l          *launcher.Launcher
-	listenURL  string
 	previewURL string
 	authKey    string
 
@@ -51,15 +52,17 @@ type Server struct {
 	// timeouts
 	generateTimeout time.Duration
 	cacheTTL        time.Duration
+
+	// browser for navigating to pages
+	browser *rod.Browser
 }
 
-func New(fs embed.FS, connOpt redis.UniversalOptions, listenURL, launcherURL, previewURL, authKey string) (*Server, error) {
+func New(fs embed.FS, connOpt redis.UniversalOptions, launcherURL, previewURL, authKey string) (*Server, error) {
 	s := &Server{
 		staticFS:        fs,
 		work:            make(chan string),
 		l:               launcher.MustNewManaged(launcherURL),
-		router:          chi.NewRouter(),
-		listenURL:       listenURL,
+		Mux:             chi.NewRouter(),
 		previewURL:      previewURL,
 		generateTimeout: 90 * time.Second,
 		cacheTTL:        15 * time.Minute,
@@ -70,6 +73,11 @@ func New(fs embed.FS, connOpt redis.UniversalOptions, listenURL, launcherURL, pr
 	if err != nil {
 		return nil, fmt.Errorf("redis ping failed: %w", err)
 	}
+	s.browser = rod.New().Client(s.l.MustClient())
+	err = s.browser.Connect()
+	if err != nil {
+		return nil, fmt.Errorf("error connecting to browser: %w", err)
+	}
 
 	return s, nil
 }
@@ -84,7 +92,7 @@ func (s *Server) SetOpts(opts ...serverCfg) error {
 	return nil
 }
 
-func (s *Server) Start() error {
+func (s *Server) Init() error {
 	if s.logger == nil {
 		s.logger = slog.New(slog.NewTextHandler(os.Stdout, nil))
 	}
@@ -94,7 +102,11 @@ func (s *Server) Start() error {
 	}
 	go s.listen()
 
-	return http.ListenAndServe(s.listenURL, s.router)
+	return nil
+}
+
+func (s *Server) Shutdown() error {
+	return s.browser.Close()
 }
 
 func WithLogger(logger *slog.Logger) serverCfg {
@@ -154,10 +166,10 @@ func WithGenerateTimeout(timeout int) serverCfg {
 }
 
 func (s *Server) routes() error {
-	s.router.Use(middleware.Logger)
-	s.router.Use(middleware.Recoverer)
-	s.router.Use(middleware.RequestID)
-	s.router.With(s.authKeyCheck).Route("/", func(r chi.Router) {
+	s.Use(middleware.Logger)
+	s.Use(middleware.Recoverer)
+	s.Use(middleware.RequestID)
+	s.With(s.authKeyCheck).Route("/", func(r chi.Router) {
 		if s.useLocalAssets {
 			localAssetsFS := http.FileServer(http.Dir(s.assetsDir))
 			r.Handle(fmt.Sprintf("%v/*", s.assetsPrefix), http.StripPrefix(s.assetsPrefix+"/", localAssetsFS))