From 4442a58766d5b96199bc410565d399f93494c9d9 Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Wed, 24 May 2023 01:07:40 +0200 Subject: [PATCH] feat: api cache system with dragonfly (#436) **Describe the pull request** This Pull Request introduces a new feature to our API - a cache system implemented with Dragonfly. The cache system is primarily used to cache all proxies from the intranet, aiming to prevent bursts on the intra API. By caching frequently requested data, the system reduces the load on the intra API, resulting in improved performance and response time. Dragonfly, a robust caching library, is utilized to handle the caching mechanism effectively, providing features such as cache expiration and invalidation. The cache system is designed to gracefully handle cache expiration and ensure that the most up-to-date data is served to the users. **Checklist** - [ ] I have linked the relative issue to this pull request - [ ] I have made the modifications or added tests related to my PR - [ ] I have added/updated the documentation for my RP - [ ] I put my PR in Ready for Review only when all the checklist is checked **Breaking changes ?** no --- .devcontainer/Dockerfile | 10 +- .devcontainer/docker-compose.yaml | 13 ++ .devcontainer/postStartCommand.sh | 2 +- .github/workflows/linters.yaml | 2 +- .github/workflows/tests.yaml | 2 +- .vscode/launch.json | 23 ++- .vscode/tasks.json | 17 ++ build/Dockerfile | 2 +- cmd/api.go | 35 ++-- cmd/campus.go | 9 - cmd/locations.go | 11 - cmd/reindexusers.go | 10 - cmd/root.go | 26 +++ deploy/stacks/apps/s42/api.tf | 8 + deploy/stacks/apps/s42/storages.tf | 100 +++++++++ generate.go | 12 +- go.mod | 13 +- go.sum | 30 ++- internal/api/api.resolvers.go | 13 +- internal/api/resolver.go | 5 +- internal/models/client.go | 7 +- .../models/templates/marshal_binary.go.tmpl | 38 ++++ internal/webhooks/serve.go | 4 - pkg/cache/client.go | 172 ++++++++++++++++ pkg/cache/gql.go | 48 +++++ pkg/cache/keybuilder.go | 189 ++++++++++++++++++ pkg/cache/option.go | 29 +++ pkg/duoapi/user.go | 13 ++ pkg/utils/random_color.go | 1 - 29 files changed, 759 insertions(+), 85 deletions(-) create mode 100644 internal/models/templates/marshal_binary.go.tmpl create mode 100644 pkg/cache/client.go create mode 100644 pkg/cache/gql.go create mode 100644 pkg/cache/keybuilder.go create mode 100644 pkg/cache/option.go diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index eb848eb6..90865e50 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -12,13 +12,13 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends gnupg2 postgresql-client \ iputils-ping dnsutils vim htop nano sudo curl build-essential zsh wget \ - fonts-powerline tig ca-certificates software-properties-common && \ + fonts-powerline tig ca-certificates software-properties-common redis-tools && \ # Register kubectl source list - curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \ - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ + curl -fsSLo /etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \ + echo "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ # Register helm source list - curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /usr/share/keyrings/helm.gpg > /dev/null && \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list && \ + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/helm.gpg > /dev/null && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list && \ # Run install of kubectl, helm and terraform apt-get update && \ apt-get install kubectl helm && \ diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index 0b271379..b3c721d9 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -25,6 +25,7 @@ services: GO_ENV: development APP_VERSION: indev DATABASE_URL: postgresql://postgres:postgres@database:5432/s42?sslmode=disable + KEYVALUE_STORE_URL: redis://:@dragonfly:6379 AMQP_URL: amqp://rabbitmq:rabbitmq@rabbitmq:5672 CORS_ORIGIN: http://localhost:3000 SEARCHENGINE_MEILISEARCH_HOST: http://meilisearch:7700 @@ -60,6 +61,17 @@ services: # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) + dragonfly: + hostname: s42-dragonfly + image: 'docker.dragonflydb.io/dragonflydb/dragonfly' + restart: unless-stopped + ulimits: + memlock: -1 + volumes: + - dragonfly-data:/data + # Use "forwardPorts" in **devcontainer.json** to forward an workspace port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + rabbitmq: hostname: s42-rabbitmq image: ghcr.io/42atomys/s42-rabbitmq:3.10.2-management @@ -91,6 +103,7 @@ services: volumes: postgres-data: + dragonfly-data: meilisearch-data: minio-data: diff --git a/.devcontainer/postStartCommand.sh b/.devcontainer/postStartCommand.sh index a06a07f9..aa5e4571 100755 --- a/.devcontainer/postStartCommand.sh +++ b/.devcontainer/postStartCommand.sh @@ -41,7 +41,7 @@ make -f build/Makefile devcontainer-init # Create the s42-users bucket go install github.com/minio/mc@latest mc alias set s3 http://minio:9000 $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY -mc mb s3/$S3_BUCKET_USERS --ignore-existing --region europe-west1 +mc mb s3/s42-users --ignore-existing --region europe-west1 # Install and configure kubeseal go install github.com/bitnami-labs/sealed-secrets/cmd/kubeseal@v0.21.0 \ No newline at end of file diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index dfb39c31..e6649317 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -30,7 +30,7 @@ jobs: - name: Setup go uses: actions/setup-go@v4 with: - go-version: "1.18" + go-version: "1.20" check-latest: true - name: Setup protoc uses: arduino/setup-protoc@v1 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 76308a55..a07cdfd4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -126,7 +126,7 @@ jobs: - name: Setup go uses: actions/setup-go@v4 with: - go-version: "1.18" + go-version: "1.20" check-latest: true - name: Setup protoc uses: arduino/setup-protoc@v1 diff --git a/.vscode/launch.json b/.vscode/launch.json index 55de9152..a82956a0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,6 +8,9 @@ "mode": "debug", "program": "${workspaceFolder}/main.go", "args": ["serve", "api", "-g"], + "env": { + "DEBUG": "true" + }, "showLog": true }, { @@ -16,7 +19,10 @@ "request": "launch", "mode": "debug", "program": "${workspaceFolder}/main.go", - "args": ["serve", "jwtks"] + "args": ["serve", "jwtks"], + "env": { + "DEBUG": "true" + } }, { "name": "Launch Interface (with chrome debug)", @@ -36,7 +42,10 @@ "request": "launch", "mode": "debug", "program": "${workspaceFolder}/main.go", - "args": ["jobs", "webhooks"] + "args": ["jobs", "webhooks"], + "env": { + "DEBUG": "true" + } }, { "name": "Launch crawler (campus)", @@ -44,7 +53,10 @@ "request": "launch", "mode": "debug", "program": "${workspaceFolder}/main.go", - "args": ["jobs", "crawler", "campus"] + "args": ["jobs", "crawler", "campus"], + "env": { + "DEBUG": "true" + } }, { "name": "Launch crawler (locations)", @@ -52,7 +64,10 @@ "request": "launch", "mode": "debug", "program": "${workspaceFolder}/main.go", - "args": ["jobs", "crawler", "locations"] + "args": ["jobs", "crawler", "locations"], + "env": { + "DEBUG": "true" + } } ] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 6ab3b690..ea1589e5 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -198,6 +198,23 @@ "panel": "dedicated" } }, + { + "label": "Run dragonfly cli (redis-cli)", + "detail": "Execute redis-cli on container", + "type": "process", + "isBackground": true, + "command": "redis-cli", + "icon": { + "id": "debug-start", + "color": "terminal.ansiRed" + }, + "problemMatcher": [], + "args": ["-h", "dragonfly", "-p", "6379"], + "presentation": { + "focus": true, + "panel": "dedicated" + } + }, { "label": "Populate DB with campus", "detail": "Execute the campus crawler to populate your development database with campus (need 42 credentials)", diff --git a/build/Dockerfile b/build/Dockerfile index 5523463a..77d526a1 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,5 +1,5 @@ # GOLANG BUILD - BUILD -FROM golang:1.18 AS go-build +FROM golang:1.20 AS go-build WORKDIR /build COPY . /build diff --git a/cmd/api.go b/cmd/api.go index f961b39f..9b7fa9ac 100644 --- a/cmd/api.go +++ b/cmd/api.go @@ -2,8 +2,9 @@ package cmd import ( "bytes" + "context" "fmt" - "io/ioutil" + "io" "net/http" "os" "strings" @@ -20,11 +21,13 @@ import ( "github.com/rs/cors" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/vektah/gqlparser/v2/gqlerror" "go.opentelemetry.io/otel" "atomys.codes/stud42/internal/api" modelsutils "atomys.codes/stud42/internal/models" "atomys.codes/stud42/internal/pkg/searchengine" + "atomys.codes/stud42/pkg/cache" "atomys.codes/stud42/pkg/otelgql" ) @@ -39,27 +42,25 @@ var apiCmd = &cobra.Command{ Short: "Serve the API in production", PreRun: func(cmd *cobra.Command, args []string) { - if err := modelsutils.Connect(); err != nil { - log.Fatal().Err(err).Msg("failed to connect to database") - } - - if err := modelsutils.Migrate(); err != nil { - log.Fatal().Err(err).Msg("failed to migrate database") - } - searchengine.Initizialize() }, Run: func(cmd *cobra.Command, args []string) { tracer := otel.GetTracerProvider().Tracer("graphql-api") - srv := handler.NewDefaultServer(api.NewSchema(modelsutils.Client(), tracer)) - // srv.SetRecoverFunc(func(ctx context.Context, err interface{}) error { - // // notify bug tracker... - // log.Error().Err(err.(error)).Msg("unhandled error") - // return gqlerror.Errorf("Internal server error!") - // }) + cacheClient, _ := cmd.Context().Value(keyValueCtxKey{}).(*cache.Client) + gqlCacheClient, err := cacheClient.NewGQLCache(30 * time.Minute) + if err != nil { + log.Fatal().Err(err).Msg("failed to init gql cache") + } + srv := handler.NewDefaultServer(api.NewSchema(modelsutils.Client(), cacheClient, tracer)) + srv.SetRecoverFunc(func(ctx context.Context, err interface{}) error { + // notify bug tracker... + log.Error().Err(err.(error)).Msg("unhandled api error") + return gqlerror.Errorf("Internal server error!") + }) srv.Use(entgql.Transactioner{TxOpener: modelsutils.Client()}) + srv.Use(extension.AutomaticPersistedQuery{Cache: gqlCacheClient}) srv.Use(extension.FixedComplexityLimit(64)) srv.Use(otelgql.Middleware(tracer)) @@ -96,7 +97,7 @@ var apiCmd = &cobra.Command{ const _50KB = (1 << 10) * 50 limitedBody := http.MaxBytesReader(w, r.Body, _50KB) - bodyBytes, err := ioutil.ReadAll(limitedBody) + bodyBytes, err := io.ReadAll(limitedBody) limitedBody.Close() // if r.Body reach the max size limit, the request will be canceled @@ -106,7 +107,7 @@ var apiCmd = &cobra.Command{ return } - r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) h.ServeHTTP(w, r) } return http.HandlerFunc(fn) diff --git a/cmd/campus.go b/cmd/campus.go index 71f72854..1691aefc 100644 --- a/cmd/campus.go +++ b/cmd/campus.go @@ -15,15 +15,6 @@ import ( var campusCmd = &cobra.Command{ Use: "campus", Short: "Crawl all campus of 42 network and update the database", - PreRun: func(cmd *cobra.Command, args []string) { - if err := modelsutils.Connect(); err != nil { - log.Fatal().Err(err).Msg("failed to connect to database") - } - - if err := modelsutils.Migrate(); err != nil { - log.Fatal().Err(err).Msg("failed to migrate database") - } - }, Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Start the crawling of all campus of 42 network") campuses, err := duoapi.CampusAll(cmd.Context()) diff --git a/cmd/locations.go b/cmd/locations.go index aa799ba3..69024c14 100644 --- a/cmd/locations.go +++ b/cmd/locations.go @@ -15,7 +15,6 @@ import ( "atomys.codes/stud42/internal/models/generated/campus" "atomys.codes/stud42/internal/models/generated/location" "atomys.codes/stud42/internal/models/generated/user" - "atomys.codes/stud42/internal/pkg/searchengine" "atomys.codes/stud42/pkg/duoapi" ) @@ -25,16 +24,6 @@ var locationsCmd = &cobra.Command{ Short: "Crawl all active locations of specific campus and update the database", Long: `Crawl all active locations of specific campus and update the database. For any closed locations, the location will be marked as inactive in the database.`, - PreRun: func(cmd *cobra.Command, args []string) { - if err := modelsutils.Connect(); err != nil { - log.Fatal().Err(err).Msg("failed to connect to database") - } - - if err := modelsutils.Migrate(); err != nil { - log.Fatal().Err(err).Msg("failed to migrate database") - } - searchengine.Initizialize() - }, Run: func(cmd *cobra.Command, args []string) { var campusID = cmd.Flag("campus_id").Value.String() db := modelsutils.Client() diff --git a/cmd/reindexusers.go b/cmd/reindexusers.go index 80d7a3b1..6638c07c 100644 --- a/cmd/reindexusers.go +++ b/cmd/reindexusers.go @@ -22,16 +22,6 @@ This operation is useful when the meilisearch index is corrupted or when the meilisearch index is not up to date. This operation will take a long time to complete. This operation will drop the meilisearch index and recreate it with all the users.`, - PreRun: func(cmd *cobra.Command, args []string) { - if err := modelsutils.Connect(); err != nil { - log.Fatal().Err(err).Msg("failed to connect to database") - } - - if err := modelsutils.Migrate(); err != nil { - log.Fatal().Err(err).Msg("failed to migrate database") - } - searchengine.Initizialize() - }, Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Prepare the re-indexation of the users") diff --git a/cmd/root.go b/cmd/root.go index 0215ea96..7820888e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,8 @@ import ( "context" "strings" + modelsutils "atomys.codes/stud42/internal/models" + "atomys.codes/stud42/pkg/cache" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -11,10 +13,34 @@ import ( var cfgFile string +type keyValueCtxKey struct{} + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "api", Short: "stud42 API", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + var cacheClient *cache.Client + var err error + + keyValueStoreUrl := viper.GetString("keyvalue-store-url") + if keyValueStoreUrl != "" { + cacheClient, err = cache.NewClient(viper.GetString("keyvalue-store-url")) + if err != nil { + log.Fatal().Err(err).Msg("failed to create cache") + } + + cmd.SetContext(context.WithValue(cmd.Context(), keyValueCtxKey{}, cacheClient)) + } + + if modelsutils.Connect(cacheClient) != nil { + log.Fatal().Msg("Failed to connect to database") + } + + if err := modelsutils.Migrate(); err != nil { + log.Fatal().Err(err).Msg("failed to migrate database") + } + }, } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/deploy/stacks/apps/s42/api.tf b/deploy/stacks/apps/s42/api.tf index 0d27c689..0dc902f0 100644 --- a/deploy/stacks/apps/s42/api.tf +++ b/deploy/stacks/apps/s42/api.tf @@ -51,6 +51,9 @@ module "api" { DATABASE_HOST = "postgres.${var.namespace}.svc.cluster.local" DATABASE_NAME = "s42" DATABASE_URL = "postgresql://postgres:$(DATABASE_PASSWORD)@$(DATABASE_HOST):5432/$(DATABASE_NAME)?sslmode=disable" + KEYVALUE_STORE_HOST = "dragonfly.${var.namespace}.svc.cluster.local" + KEYVALUE_STORE_PORT = "6379" + KEYVALUE_STORE_URL = "redis://:$(DFLY_PASSWORD)@$(KEYVALUE_STORE_HOST):$(KEYVALUE_STORE_PORT)" SEARCHENGINE_MEILISEARCH_HOST = "http://meilisearch.${var.namespace}.svc.cluster.local:7700" } @@ -60,6 +63,11 @@ module "api" { name = "postgres-credentials" } + DFLY_PASSWORD = { + key = "DFLY_PASSWORD" + name = "dragonfly-credentials" + } + GITHUB_TOKEN = { key = "GITHUB_TOKEN" name = "github-token" diff --git a/deploy/stacks/apps/s42/storages.tf b/deploy/stacks/apps/s42/storages.tf index 84989a9c..cd6fd407 100644 --- a/deploy/stacks/apps/s42/storages.tf +++ b/deploy/stacks/apps/s42/storages.tf @@ -3,6 +3,13 @@ locals { rabbitmqClusterReference = { name = local.rabbitmqClusterName } + + dragonflyProbe = { + httpGet = { + path = "/" + port = "dragonfly" + } + } } resource "kubernetes_manifest" "rabbitmq" { @@ -219,3 +226,96 @@ module "postgres" { } } : {} } + +resource "random_password" "dragonfly" { + length = 64 + special = true +} + +module "dragonfly" { + source = "../../../modules/service" + kind = "StatefulSet" + + appName = "dragonfly" + appVersion = "v1.3.0" + name = "dragonfly" + namespace = var.namespace + image = "docker.dragonflydb.io/dragonflydb/dragonfly:v1.3.0" + imagePullPolicy = "IfNotPresent" + + nodeSelector = local.nodepoolSelector["storages"] + + revisionHistoryLimit = 1 + replicas = 1 + maxUnavailable = 0 + + prometheus = { + enabled = true + port = 6379 + } + + livenessProbe = local.dragonflyProbe + readinessProbe = local.dragonflyProbe + + resources = { + requests = { + cpu = "100m" + memory = "128Mi" + } + + limits = { + memory = "256Mi" + } + } + + env = {} + + envFromSecret = { + DFLY_PASSWORD = { + key = "DFLY_PASSWORD" + name = "dragonfly-credentials" + } + } + + ports = { + dragonfly = { + containerPort = 6379 + istioProtocol = "tcp" + } + } + + volumeMounts = [ + { + mountPath = "/data" + volumeName = "data" + } + ] + + volumesFromPVC = var.hasPersistentStorage ? { + data = { + claimName = "dragonfly-data" + readOnly = false + } + } : {} + + volumesFromEmptyDir = !var.hasPersistentStorage ? { + data = {} + } : {} + + secrets = { + credentials = { + data = { + "DFLY_PASSWORD" = random_password.dragonfly.result + "DFLY_PASSWORD_ENCODED" = urlencode(random_password.dragonfly.result) + } + } + } + + persistentVolumeClaims = var.hasPersistentStorage ? { + data = { + accessModes = ["ReadWriteMany"] + storage = "2Gi" + storageClassName = "csi-cinder-high-speed" + } + } : {} +} diff --git a/generate.go b/generate.go index aa1c2a80..53032106 100644 --- a/generate.go +++ b/generate.go @@ -10,6 +10,7 @@ import ( "log" "os" + "atomys.codes/stud42/pkg/cache" "entgo.io/contrib/entgql" "entgo.io/ent/entc" "entgo.io/ent/entc/gen" @@ -35,6 +36,15 @@ func generateEntc() { log.Fatalf("creating entgql extension: %v", err) } + opts := []entc.Option{ + entc.Extensions(ex), + entc.TemplateDir("./internal/models/templates"), + entc.Dependency( + entc.DependencyName("Cache"), + entc.DependencyType(&cache.Client{}), + ), + } + err = entc.Generate("./internal/models/schema", &gen.Config{ Features: []gen.Feature{ gen.FeaturePrivacy, @@ -45,7 +55,7 @@ func generateEntc() { }, Target: "./internal/models/generated", Package: "atomys.codes/stud42/internal/models/generated", - }, entc.Extensions(ex)) + }, opts...) if err != nil { log.Fatalf("running ent codegen: %v", err) } diff --git a/go.mod b/go.mod index 5c6664b2..0b1635e4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module atomys.codes/stud42 -go 1.18 +go 1.20 require ( entgo.io/contrib v0.3.0 @@ -17,6 +17,7 @@ require ( github.com/lib/pq v1.10.7 github.com/meilisearch/meilisearch-go v0.21.1 github.com/pkg/errors v0.9.1 + github.com/redis/go-redis/v9 v9.0.4 github.com/rs/cors v1.6.0 github.com/rs/zerolog v1.26.1 github.com/spf13/cobra v1.6.1 @@ -29,7 +30,7 @@ require ( go.opentelemetry.io/otel/exporters/jaeger v1.9.0 go.opentelemetry.io/otel/sdk v1.9.0 go.opentelemetry.io/otel/trace v1.9.0 - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 ) @@ -40,8 +41,10 @@ require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -69,10 +72,11 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect @@ -82,12 +86,13 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fd70d873..698d1653 100644 --- a/go.sum +++ b/go.sum @@ -59,10 +59,14 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/aws/aws-sdk-go v1.44.261 h1:PcTMX/QVk+P3yh2n34UzuXDF5FS2z5Lse2bt+r3IpU4= github.com/aws/aws-sdk-go v1.44.261/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bwmarrin/discordgo v0.25.0 h1:NXhdfHRNxtwso6FPdzW2i3uBvvU7UIQTghmV2T4nqAs= github.com/bwmarrin/discordgo v0.25.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -82,6 +86,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -91,6 +97,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= @@ -209,8 +216,8 @@ github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -240,6 +247,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzC github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -250,8 +259,11 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= +github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -262,8 +274,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -402,18 +414,19 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -622,8 +635,9 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/api/api.resolvers.go b/internal/api/api.resolvers.go index 37d42e8b..a2e3d89c 100644 --- a/internal/api/api.resolvers.go +++ b/internal/api/api.resolvers.go @@ -25,6 +25,7 @@ import ( "atomys.codes/stud42/internal/models/generated/user" "atomys.codes/stud42/internal/models/gotype" "atomys.codes/stud42/internal/pkg/searchengine" + "atomys.codes/stud42/pkg/cache" "atomys.codes/stud42/pkg/duoapi" "atomys.codes/stud42/pkg/utils" "entgo.io/ent/dialect/sql" @@ -625,12 +626,14 @@ func (r *userResolver) IsSwimmer(ctx context.Context, obj *generated.User) (bool // IntraProxy is the resolver for the intraProxy field. func (r *userResolver) IntraProxy(ctx context.Context, obj *generated.User) (*duoapi.User, error) { - intraUser, err := duoapi.UserGet(ctx, strconv.Itoa(obj.DuoID)) - if err != nil { - return nil, err - } + cacheKey := cache.NewKeyBuilder().WithPrefix("intra-proxy").WithObject(obj.ID).Build() + + loader := cache.New[*duoapi.User](r.cache).WithLoader(func(ctx context.Context, key cache.CacheKey) (*duoapi.User, error) { + return duoapi.UserGet(ctx, strconv.Itoa(obj.DuoID)) + }) + defer loader.Close() - return intraUser, nil + return loader.Get(ctx, cacheKey, cache.WithExpiration(24*time.Hour)) } // FollowersCount is the resolver for the followersCount field. diff --git a/internal/api/resolver.go b/internal/api/resolver.go index df85bd16..f5840ab4 100644 --- a/internal/api/resolver.go +++ b/internal/api/resolver.go @@ -6,6 +6,7 @@ import ( apigen "atomys.codes/stud42/internal/api/generated" modelgen "atomys.codes/stud42/internal/models/generated" + "atomys.codes/stud42/pkg/cache" ) // This file will not be regenerated automatically. @@ -14,16 +15,18 @@ import ( type Resolver struct { client *modelgen.Client + cache *cache.Client tracer trace.Tracer } type contextKey string // NewSchema creates a graphql executable schema. -func NewSchema(client *modelgen.Client, tr trace.Tracer) graphql.ExecutableSchema { +func NewSchema(client *modelgen.Client, cacheClient *cache.Client, tr trace.Tracer) graphql.ExecutableSchema { return apigen.NewExecutableSchema(apigen.Config{ Resolvers: &Resolver{ client: client, + cache: cacheClient, tracer: tr, }, Directives: apigen.DirectiveRoot{ diff --git a/internal/models/client.go b/internal/models/client.go index 3a18a736..2b4a35a8 100644 --- a/internal/models/client.go +++ b/internal/models/client.go @@ -8,18 +8,23 @@ import ( modelgen "atomys.codes/stud42/internal/models/generated" _ "atomys.codes/stud42/internal/models/generated/runtime" + "atomys.codes/stud42/pkg/cache" ) var client *modelgen.Client // Connect to the database and create the client. -func Connect() (err error) { +func Connect(cacheClient *cache.Client) (err error) { var opts = []modelgen.Option{} if os.Getenv("DEBUG") == "true" { opts = append(opts, modelgen.Debug()) } + if cacheClient != nil { + opts = append(opts, modelgen.Cache(cacheClient)) + } + client, err = modelgen.Open( "postgres", os.Getenv("DATABASE_URL"), diff --git a/internal/models/templates/marshal_binary.go.tmpl b/internal/models/templates/marshal_binary.go.tmpl new file mode 100644 index 00000000..a97de07e --- /dev/null +++ b/internal/models/templates/marshal_binary.go.tmpl @@ -0,0 +1,38 @@ +{{/* gotype: entgo.io/ent/entc/gen.Graph */}} + +{{ define "import/additional/marshal_binary" }} + "bytes" + "encoding/gob" +{{ end }} + +{{ define "model/additional/marshal_binary" }} + +func ({{ $.Receiver }} *{{ $.Name }}) MarshalBinary() ([]byte, error) { + return json.Marshal({{ $.Receiver }}) +} + +func ({{ $.Receiver }} *{{ $.Name }}) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, {{ $.Receiver }}) +} + +{{ end }} + +{{ define "gql_pagination_marshal" }} + {{ template "header" $ }} + + {{ range $n := $.Nodes }} + {{ template "gql_pagination/helper/marshal_binary" $n }} + {{ end }} +{{ end }} + +{{ define "gql_pagination/helper/marshal_binary" }} + +func ({{ $.Receiver }} *{{ $.Name }}Connection) MarshalBinary() ([]byte, error) { + return json.Marshal({{ $.Receiver }}) +} + +func ({{ $.Receiver }} *{{ $.Name }}Connection) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, {{ $.Receiver }}) +} + +{{ end }} \ No newline at end of file diff --git a/internal/webhooks/serve.go b/internal/webhooks/serve.go index 76a820fc..c2872893 100644 --- a/internal/webhooks/serve.go +++ b/internal/webhooks/serve.go @@ -32,10 +32,6 @@ var ErrInvalidWebhook = errors.New("invalid webhook, metadata is empty") // New creates a new webhooks processor instance func New() *processor { - if err := modelsutils.Connect(); err != nil { - log.Fatal().Err(err).Msg("failed to connect to database") - } - return &processor{ ctx: context.Background(), db: modelsutils.Client(), diff --git a/pkg/cache/client.go b/pkg/cache/client.go new file mode 100644 index 00000000..eb35f346 --- /dev/null +++ b/pkg/cache/client.go @@ -0,0 +1,172 @@ +package cache + +import ( + "context" + "encoding" + "errors" + "reflect" + "sync" + "time" + + "log" + + "github.com/redis/go-redis/v9" +) + +type Client struct { + redisStore *redis.Client +} + +type LoadFunction[T any] func(ctx context.Context, key CacheKey) (T, error) + +type loadableKeyValue[T any] struct { + key CacheKey + value T + options []option +} + +type TypedClient[T any] struct { + *Client + setterWg *sync.WaitGroup + setChannel chan *loadableKeyValue[T] + loader LoadFunction[T] +} + +// New create a new cache client with initialized stores and cache. It will +// return an error if the cache cannot be initialized. +// +// The cache will be initialized with the following stores: +// - Redis: Redis cache for distributed cache. +func NewClient(redisUrl string) (*Client, error) { + // Redis is a high performance key value store that is used for caching + // this is used as fallback to ristretto cache when we are in a high + // traffic situation (scale on multiple instances). + opts, err := redis.ParseURL(redisUrl) + if err != nil { + return nil, err + } + + opts.DialTimeout = 10 * time.Second + opts.ReadTimeout = 30 * time.Second + opts.WriteTimeout = 30 * time.Second + opts.PoolSize = 10 + opts.PoolTimeout = 30 * time.Second + + redisClient := redis.NewClient(opts) + if _, err := redisClient.Ping(context.Background()).Result(); err != nil { + return nil, err + } + + return &Client{ + redisStore: redisClient, + }, nil +} + +type BinaryMarshable interface { + encoding.BinaryMarshaler + encoding.BinaryUnmarshaler +} + +var ErrNotFound = errors.New("item not found in cache") + +// New creates a new typed cache client with initialized stores and cache. +// The type is used to ensure that the cache is only used for the correct type +// and automatically casted to the correct type. +func New[T any](client *Client) *TypedClient[T] { + + typedClient := &TypedClient[T]{ + Client: client, + loader: func(ctx context.Context, key CacheKey) (obj T, err error) { + return obj, ErrNotFound + }, + setterWg: &sync.WaitGroup{}, + setChannel: make(chan *loadableKeyValue[T]), + } + + typedClient.setterWg.Add(1) + go typedClient.setter() + + return typedClient +} + +func (tc *TypedClient[T]) setter() { + defer tc.setterWg.Done() + + for item, ok := <-tc.setChannel; ok; item, ok = <-tc.setChannel { + if err := tc.Set(context.Background(), item.key, item.value, item.options...); err != nil { + log.Printf("failed to set value in cache: %s", err.Error()) + } + } +} + +// Close the LoadableCache update channel to prevent any goroutine leak +// and wait for the setter goroutine to finish +func (c *TypedClient[T]) Close() error { + close(c.setChannel) + + return nil +} + +// WithLoader is a wrapper around the cache that allows us to load data from the +// `lf` function if it is not in the cache. This is useful for loading data +// from a database or other source. +func (tc *TypedClient[T]) WithLoader(lf LoadFunction[T]) *TypedClient[T] { + tc.loader = lf + return tc +} + +// Get returns the value stored in the cache for the given key. If the key is +// not found in the cache, and this function is chained with `Loader` before +// the value is loaded from the `lf` function and stored in the cache. If the +// key is not found in the cache and there is no `Loader` function, then the +// value is nil and the bool is false. +func (tc *TypedClient[T]) Get(ctx context.Context, key CacheKey, setOptions ...option) (object T, err error) { + if reflect.ValueOf(object).Kind() != reflect.Ptr { + return object, errors.New("object must be a pointer") + } + + // initialize the object to the zero value of the type + object = reflect.New(reflect.TypeOf(object).Elem()).Interface().(T) + + err = tc.redisStore.Get(ctx, key.String()).Scan(object) + if err == nil { + return object, nil + } + + if !errors.Is(err, redis.Nil) { + return object, err + } + + object, err = tc.loader(ctx, key) + if err == nil { + tc.setChannel <- &loadableKeyValue[T]{ + key: key, + value: object, + options: setOptions, + } + } + + return object, err +} + +// Set stores the value in the cache for the given key. The value is stored +// with the given options. +func (tc *TypedClient[T]) Set(ctx context.Context, key CacheKey, object T, options ...option) error { + o := ApplyOptions(options...) + return tc.redisStore.Set(ctx, key.String(), object, o.Expiration).Err() +} + +// Delete removes the value from the cache for the given key. +func (tc *TypedClient[T]) Delete(ctx context.Context, key CacheKey) error { + return tc.redisStore.Del(ctx, key.String()).Err() +} + +// // Invalidate removes all the values from the cache for the given tags. +// // If no tags are given, all the values are removed from the cache. +// func (tc *TypedClient[T]) Invalidate(ctx context.Context, options ...store.InvalidateOption) error { +// } + +// Clear removes all the values from the cache. +func (tc *TypedClient[T]) Clear(ctx context.Context) error { + return tc.redisStore.FlushAll(ctx).Err() +} diff --git a/pkg/cache/gql.go b/pkg/cache/gql.go new file mode 100644 index 00000000..4fbae62f --- /dev/null +++ b/pkg/cache/gql.go @@ -0,0 +1,48 @@ +package cache + +import ( + "context" + "log" + "time" +) + +type GQLCache struct { + *TypedClient[any] + ttl time.Duration +} + +var ( + // gqlCacheKey is the key used to store the query cache usef internally on + // NewGQLCache type + gqlCacheKey = NewKeyBuilder().WithPrefix("s42-gql-cache") +) + +// NewGQLCache creates a new cache client for the graphql layer with the given +// ttl for the cache. +func (c *Client) NewGQLCache(ttl time.Duration) (*GQLCache, error) { + return &GQLCache{TypedClient: New[any](c), ttl: ttl}, nil +} + +// Add adds a new value to the cache with the given key. +// This is based on the [GQLGen Doc] +// +// [GQLGen Doc]: https://github.com/99designs/gqlgen/blob/master/graphql/cache.go +func (c *GQLCache) Add(ctx context.Context, key string, value interface{}) { + err := c.Set(ctx, gqlCacheKey.WithObject(key).Build(), value, WithExpiration(c.ttl)) + if err != nil { + log.Printf("failed to add to cache: %s", err.Error()) + } +} + +// Get gets a value from the cache with the given key. +// This is based on the [GQLGen Doc] +// +// [GQLGen Doc]: https://github.com/99designs/gqlgen/blob/master/graphql/cache.go +func (c *GQLCache) Get(ctx context.Context, key string) (interface{}, bool) { + s, err := c.TypedClient.Get(ctx, gqlCacheKey.WithObject(key).Build()) + if err != nil { + return struct{}{}, false + } + + return s, true +} diff --git a/pkg/cache/keybuilder.go b/pkg/cache/keybuilder.go new file mode 100644 index 00000000..20224030 --- /dev/null +++ b/pkg/cache/keybuilder.go @@ -0,0 +1,189 @@ +package cache + +/** + * The cache package is used to manage the multiple cache system for s42 app. + * + * The key builder is used to generate the key of the cache with a simplify + * way to manage it. A key must be have a scope (prefix), multiple parts (parts) + * and a suffix. The key builder is used to generate the key with a separator + * between each part of the key. The default separator is a colon and can be + * changed with `WithSeparator` function. + * + * Examples of build : + * - WithPrefix("user").WithObject("1").WithSuffix("profile").Build() => user:1:profile + * - WithPrefix("user").WithObject("1").WithParts("profile", "avatar").Build() => user:1:profile:avatar + * - WithPrefix("user").WithObject("1").WithParts("profile", "avatar").WithSuffix("outdated").Build() => user:1:profile:avatar:outdated + * + * The most important rule when you use the key builder is to use the same + * separator in all your application. If you change the separator, you must + * change it everywhere in your application. + * + * When you use the key builder, you must use the `WithPrefix` function to + * set the scope of the key. The prefix is used to know where the key is used. + * For example, if you use the key builder to generate a key for the current user + * profile, you must use the prefix `s42-current-user` (KeyBuilder is stored in + * the `CurrentUserCacheKey` variable below). + * + * THE PREFIX, SUFFIX, PARTS AND KEY MUST BE ALWAYS IN KEBAB CASE (kebab-case) + */ + +import ( + "fmt" + "regexp" + "strings" + + "log" +) + +type CacheKey string + +const ( + // defaultKeySeparator is the key separator used on all application. + // WARNING: If you change the separator, you must change it everywhere in + // your application. + defaultKeySeparator = ":" +) + +// String will return the cache key as string to matche the Stringer interface +func (k CacheKey) String() string { + return string(k) +} + +// KeyBuilder is used to build a cache key with a prefix, a suffix, multiple +// parts and a key. +type KeyBuilder struct { + key string + parts []string + prefix string + suffix string + separator string +} + +var ( + // The followings vars is used to transform any style to kebak-style + // to ensure the kebak style + matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") + matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") + keyReplacer = strings.NewReplacer(".", " ", "_", " ", "-", " ") +) + +// String will generate and return the builded key as String (we use String to +// follow the Stringer interface of Golang). +// +// String method will ensure the kebab style of the key. In case of the key is +// not in kebab style, the key will be converted to kebab style and print an +// Warning to the log +func (b *KeyBuilder) String() string { + var keyParts = []string{} + + if b.prefix != "" { + keyParts = append(keyParts, kebabCase(b.prefix)) + } + + if b.key != "" { + keyParts = append(keyParts, kebabCase(string(b.key))) + } + + if len(b.parts) > 0 { + for _, part := range b.parts { + keyParts = append(keyParts, kebabCase(part)) + } + } + + if b.suffix != "" { + keyParts = append(keyParts, kebabCase(b.suffix)) + } + + return strings.Join(keyParts, b.separator) +} + +// WithPrefix will set the prefix of the key. The prefix must be in +// kebab-case. When your prefix is used in multiple place (like s42-current-user) +// add the pre-builded key on the var group on top of this file to prevent code +// duplication +func (b *KeyBuilder) WithPrefix(prefix string) *KeyBuilder { + b.prefix = prefix + return b +} + +// WithSuffix will set the suffix of the key. The suffix must be in +// kebab-case. When your suffix is used in multiple place add the pre-builded +// key on the var group on top of this file to prevent code duplication +func (b *KeyBuilder) WithSuffix(suffix string) *KeyBuilder { + b.suffix = suffix + return b +} + +// WithObject will set the object of the key. This is used principally to define +// the object relative to the cacke key. The Object must be a string or a +// Stringer (implement the `String() string` method). +// Example, when you build the cache key for current user : +// +// CurrentUserCacheKey.WithObject("00000000-0000-0000-0000-000000000000").Build() +// // "s42-current-user:00000000-0000-0000-0000-000000000000" +func (b *KeyBuilder) WithObject(key any) *KeyBuilder { + switch s := key.(type) { + case fmt.Stringer: + b.key = s.String() + case string: + b.key = s + default: + log.Printf("WARN: The key %v is not a stringer or a string", key) + } + return b +} + +// WithParts will set extra parts on your key to store extra informations about +// your object. +// Example, store avatar object of the current user in small size : +// +// CurrentUserCacheKey +// .WithObject("00000000-0000-0000-0000-000000000000") +// .WithParts("avatar", "small") +// .Build() // "s42-current-user:00000000-0000-0000-0000-000000000000:avatar:small" +func (b *KeyBuilder) WithParts(parts ...string) *KeyBuilder { + b.parts = parts + return b +} + +// WithSeparator will set the separator used to build the key. The default +// separator is a colon. If you change the separator, you must change it +// everywhere in your application. Be carreful when you change the separator +func (b *KeyBuilder) WithSeparator(separator string) *KeyBuilder { + b.separator = separator + return b +} + +// NewKeyBuilder will return a new KeyBuilder with the default separator +// (colon). Always start your KeyBuilder journey here. +func NewKeyBuilder() *KeyBuilder { + return &KeyBuilder{separator: defaultKeySeparator} +} + +// Build will transform the key builder to a CacheKey type to matche the +// Stringer interface of Golang and to be used in the cache package of the +// application +func (b *KeyBuilder) Build() CacheKey { + return CacheKey(b.String()) +} + +// kebakCase will transform any style to kebak-style to ensure the kebak style +// as the convention of the cache. In case of the key is not in kebab style, the +// key will be converted to kebab style and print an Warning to the log. +func kebabCase(s string) string { + // convert camel case to snake case + s = matchAllCap.ReplaceAllString( + matchFirstCap.ReplaceAllString(s, "${1} ${2}"), + "${1} ${2}", + ) + + if strings.Contains(s, " ") { + log.Printf("WARN: Your cache key is not in kebak-case. Please convert it to avoid any issue. The key is %s", s) + } + + // convert to lower case and replace separators with spaces + s = keyReplacer.Replace(s) + s = strings.Join(strings.Fields(s), "-") + + return strings.ToLower(s) +} diff --git a/pkg/cache/option.go b/pkg/cache/option.go new file mode 100644 index 00000000..6709ae2a --- /dev/null +++ b/pkg/cache/option.go @@ -0,0 +1,29 @@ +package cache + +import "time" + +type option func(*SetOption) + +type SetOption struct { + Expiration time.Duration +} + +// WithExpiration sets the expiration time for the cache key. +// If the expiration is set to 0, the key will never expire. +func WithExpiration(expiration time.Duration) option { + return func(o *SetOption) { + o.Expiration = expiration + } +} + +// ApplyOptions applies the given options to the SetOption struct. +// If no options are given, the default options are used. +func ApplyOptions(opts ...option) SetOption { + o := SetOption{ + Expiration: 0, + } + for _, opt := range opts { + opt(&o) + } + return o +} diff --git a/pkg/duoapi/user.go b/pkg/duoapi/user.go index ea3608bf..773af10d 100644 --- a/pkg/duoapi/user.go +++ b/pkg/duoapi/user.go @@ -2,6 +2,7 @@ package duoapi import ( "context" + "encoding/json" ) func UserGet(ctx context.Context, userID string) (*User, error) { @@ -48,3 +49,15 @@ func (l *User) ProcessWebhook(ctx context.Context, metadata *WebhookMetadata, pr } return nil } + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +// This is used by the cache package. +func (obj *User) MarshalBinary() ([]byte, error) { + return json.Marshal(obj) +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// This is used by the cache package. +func (obj *User) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, &obj) +} diff --git a/pkg/utils/random_color.go b/pkg/utils/random_color.go index df2286c2..7ebcee28 100644 --- a/pkg/utils/random_color.go +++ b/pkg/utils/random_color.go @@ -24,7 +24,6 @@ func GetRandomRBGColor() *RGBColor { func GetRandomHexColor() string { bytes := make([]byte, 3) - rand.Seed(time.Now().UnixNano()) _, err := randOnUnix().Read(bytes) if err != nil { panic(err)