Feat: Caching (#18)

* move minio to a seperate file

* feat: rudimentary impl of from cache

* ci: update versions

* feat: add storage cleaning job

* docs: update readme for self hosting config

* docs: typo

.env.example

@@ -1,6 +1,10 @@
-MINIO_ROOT_USER="" # eg: minioadmin
-MINIO_ROOT_PASSWORD="" # eg: "minioadmin" 
-BUCKET_NAME="" # eg: "binaries" 
+STORAGE_ENABLED=
+STORAGE_CLIENT_ID=
+STORAGE_CLIENT_SECRET=
+STORAGE_ENDPOINT=
+STORAGE_BUCKET=
 ORIGIN_URL="" # eg: "http://localhost" depending on your nginx config changes
-MINIO_URL_PREFIX="" # eg: "http://localhost:9000" depending on your nginx config changes
 GITHUB_TOKEN="" # required for github version resolutions
+
+# uncomment the below to enable cache clearing based on the passed duration
+# CLEAR_CACHE_TIME="10m"

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: [1.16, 1.17, '1.20.0']
+        version: ['1.19.0','1.21.5']
     steps:
       - uses: actions/checkout@v3
 

cmd/goblin-api/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -11,17 +12,18 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
+	"github.com/barelyhuman/go/env"
 	"github.com/barelyhuman/goblin/build"
 	"github.com/barelyhuman/goblin/resolver"
+	"github.com/barelyhuman/goblin/storage"
 	"github.com/joho/godotenv"
 )
 
 var shTemplates *template.Template
 var serverURL string
-
-// FIXME: Disabled storage and caching for initial version
-// var storageClient *storage.Storage
+var storageClient storage.Storage
 
 func HandleRequest(rw http.ResponseWriter, req *http.Request) {
 	path := req.URL.Path
@@ -40,6 +42,7 @@ func HandleRequest(rw http.ResponseWriter, req *http.Request) {
 	}
 
 	if strings.HasPrefix(path, "/binary") {
+		log.Print("handle binary")
 		fetchBinary(rw, req)
 		return
 	}
@@ -63,20 +66,12 @@ func StartServer(port string) {
 	}
 }
 
-func envDefault(key string, def string) string {
-	if s := os.Getenv(key); len(strings.TrimSpace(s)) == 0 {
-		return def
-	} else {
-		return s
-	}
-}
-
 // TODO: cleanup code
 // TODO: move everything into their own interface/structs
 func main() {
 
 	envFile := flag.String("env", ".env", "path to read the env config from")
-	portFlag := envDefault("PORT", "3000")
+	portFlag := env.Get("PORT", "3000")
 
 	flag.Parse()
 
@@ -88,19 +83,64 @@ func main() {
 	}
 
 	shTemplates = template.Must(template.ParseGlob("templates/*"))
-	serverURL = envDefault("ORIGIN_URL", "http://localhost:3000")
+	serverURL = env.Get("ORIGIN_URL", "http://localhost:"+portFlag)
 
-	// FIXME: Disabled storage and caching for initial version
-	// storageClient = &storage.Storage{}
-	// storageClient.BucketName = os.Getenv("BUCKET_NAME")
-	// err := storageClient.Connect()
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
+	if isStorageEnabled() {
+		storageClient = storage.NewAWSStorage(env.Get("STORAGE_BUCKET", "goblin-cache"))
+		err := storageClient.Connect()
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
 
+	clearStorageBackgroundJob()
 	StartServer(portFlag)
 }
 
+func clearStorageBackgroundJob() {
+	cacheHoldEnv := env.Get("CLEAR_CACHE_TIME", "")
+	if len(cacheHoldEnv) == 0 {
+		return
+	}
+
+	cacheHoldDuration, _ := time.ParseDuration(cacheHoldEnv)
+
+	cleaner := func(storageClient storage.Storage) {
+		log.Println("Cleaning Cached Storage Object")
+		objects := storageClient.ListObjects()
+		for _, obj := range objects {
+			objExpiry := obj.LastModified.Add(cacheHoldDuration)
+			if time.Now().Equal(objExpiry) || time.Now().After(objExpiry) {
+				storageClient.RemoveObject(obj.Key)
+			}
+		}
+	}
+
+	ticker := time.NewTicker(cacheHoldDuration)
+	quit := make(chan struct{})
+
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				cleaner(storageClient)
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}
+
+func isStorageEnabled() bool {
+	useStorageEnv := env.Get("STORAGE_ENABLED", "false")
+	useStorage := false
+	if useStorageEnv == "true" {
+		useStorage = true
+	}
+	return useStorage
+}
+
 func normalizePackage(pkg string) string {
 	// strip leading protocol
 	pkg = strings.Replace(pkg, "https://", "", 1)
@@ -228,38 +268,49 @@ func fetchBinary(rw http.ResponseWriter, req *http.Request) {
 		Module:  mod,
 	}
 
-	// TODO: check the storage for existing binary for the module
-	// and return from the storage instead
-
 	immutable(rw)
 
-	// FIXME: Disabled storage and caching for initial version
-	// var buf bytes.Buffer
-	// err := bin.WriteBuild(io.MultiWriter(rw, &buf))
+	artifactName := constructArtifactName(bin)
 
-	err := bin.WriteBuild(io.MultiWriter(rw))
+	if isStorageEnabled() && storageClient.HasObject(artifactName) {
+		url, _ := storageClient.GetSignedURL(artifactName)
+		log.Println("From cache")
+		http.Redirect(rw, req, url, http.StatusSeeOther)
+		return
+	}
+
+	var buf bytes.Buffer
+	err := bin.WriteBuild(io.MultiWriter(rw, &buf))
 
 	if err != nil {
 		rw.WriteHeader(http.StatusInternalServerError)
 		fmt.Fprint(rw, err.Error())
 		return
 	}
 
+	if isStorageEnabled() {
+		err = storageClient.Upload(
+			artifactName,
+			buf,
+		)
+
+		if err != nil {
+			log.Println("Failed to upload", err)
+		}
+	}
+
 	err = bin.Cleanup()
 	if err != nil {
 		log.Println("cleaning binary build", err)
 	}
+}
 
-	// FIXME: Disabled storage and caching for initial version
-	// err = storageClient.Upload(bin.Module, bin.Dest)
-	// if err != nil {
-	// 	fmt.Fprint(rw, err.Error())
-	// 	return
-	// }
-
-	// url, err := storageClient.GetSignedURL(bin.Module, bin.Name)
-	// if err != nil {
-	// 	fmt.Fprint(rw, err.Error())
-	// 	return
-	// }
+func constructArtifactName(bin *build.Binary) string {
+	var artifactName strings.Builder
+	artifactName.Write([]byte(bin.Name))
+	artifactName.Write([]byte("-"))
+	artifactName.Write([]byte(bin.OS))
+	artifactName.Write([]byte("-"))
+	artifactName.Write([]byte(bin.Arch))
+	return artifactName.String()
 }

docker-compose.yml

@@ -16,6 +16,25 @@ services:
       - "3000:3000"
     volumes:
       - "./:/usr/src/app"
+    profiles: [app]
+
+  minio:
+    image: "minio/minio:RELEASE.2024-05-01T01-11-10Z"
+    container_name: goblin_minio
+    command: 'server --console-address ":9001" /home/files'
+    healthcheck:
+      test: ["CMD", "curl", "-I", "http://127.0.0.1:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    env_file:
+      - .env
+    expose:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - "goblin:/data"
+    profiles: [storage]
 
   nginx:
     container_name: goblin_nginx
@@ -27,4 +46,8 @@ services:
       - "80:80"
       - "9000:9000"
     volumes:
-      - "./nginx.conf:/etc/nginx/nginx.conf:ro"
+      - "./nginx.conf:/etc/nginx/nginx.conf:ro"
+    profiles: [app]
+
+volumes:
+  goblin:

go.mod

@@ -4,35 +4,26 @@ go 1.19
 
 require (
 	github.com/Masterminds/semver v1.5.0
+	github.com/barelyhuman/go v0.2.2
 	github.com/google/go-github/v53 v53.2.0
 	github.com/joho/godotenv v1.5.1
-	github.com/minio/minio-go/v7 v7.0.66
+	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/tj/go-semver v1.0.0
 	golang.org/x/oauth2 v0.16.0
 )
 
 require (
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.1 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/rs/xid v1.5.0 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 )