feat: implement cache warmup using filesystem (#1437)

Co-authored-by: Ludwig Bedacht <[email protected]> Co-authored-by: starptech <[email protected]>
wundergraph · Dec 18, 2024 · 1efd072 · 1efd072
1 parent 286a881
commit 1efd072
Show file tree

Hide file tree

Showing 52 changed files with 2,417 additions and 332 deletions.
diff --git a/demo/go.mod b/demo/go.mod
@@ -30,6 +30,7 @@ require (
 	github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
@@ -135,6 +136,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/ratelimit v0.3.1 // indirect
 	go.withmatt.com/connect-brotli v0.4.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect

diff --git a/demo/go.sum b/demo/go.sum
@@ -17,6 +17,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
@@ -348,6 +350,8 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0=
+go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=

diff --git a/docker/grafana/provisioning/dashboards/prometheus/router_example_dashboard.json b/docker/grafana/provisioning/dashboards/prometheus/router_example_dashboard.json
@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 4,
+  "id": 2,
   "links": [],
   "panels": [
     {
@@ -88,9 +88,9 @@
         {
           "editorMode": "code",
           "exemplar": false,
-          "expr": "router_graphql_cache_cost_max{cache_type=\"execution\"}",
+          "expr": "router_graphql_cache_cost_max{cache_type=\"plan\"}",
           "instant": true,
-          "legendFormat": "Execution",
+          "legendFormat": "Plan",
           "range": false,
           "refId": "Execution"
         },
@@ -115,12 +115,25 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "router_graphql_cache_cost_max{cache_type=\"normalization\"}",
+          "expr": "router_graphql_cache_cost_max{cache_type=\"query_normalization\"}",
           "hide": false,
           "instant": true,
-          "legendFormat": "Normalization",
+          "legendFormat": "Query Normalization",
           "range": false,
           "refId": "Normalization"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "router_graphql_cache_cost_max{cache_type=\"persisted_query_normalization\"}",
+          "hide": true,
+          "instant": false,
+          "legendFormat": "Persisted Query Normalization",
+          "range": true,
+          "refId": "A"
         }
       ],
       "title": "Total Cache Sizes (In Cost)",
@@ -282,8 +295,8 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "sum(rate(router_graphql_cache_requests_stats_total{cache_type=\"execution\" ,type=\"hits\"}[2h])) /\nsum(rate(router_graphql_cache_requests_stats_total{cache_type=\"execution\"}[2h]))",
-          "legendFormat": "Execution",
+          "expr": "sum(rate(router_graphql_cache_requests_stats_total{cache_type=\"plan\" ,type=\"hits\"}[2h])) /\nsum(rate(router_graphql_cache_requests_stats_total{cache_type=\"plan\"}[2h]))",
+          "legendFormat": "Plan",
           "range": true,
           "refId": "A"
         },
@@ -306,10 +319,10 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "sum(rate(router_graphql_cache_requests_stats_total{cache_type=\"normalization\" ,type=\"hits\"}[2h])) /\nsum(rate(router_graphql_cache_requests_stats_total{cache_type=\"normalization\"}[2h]))",
+          "expr": "sum(rate(router_graphql_cache_requests_stats_total{cache_type=\"query_normalization\" ,type=\"hits\"}[2h])) /\nsum(rate(router_graphql_cache_requests_stats_total{cache_type=\"query_normalization\"}[2h]))",
           "hide": false,
           "instant": false,
-          "legendFormat": "Normalization",
+          "legendFormat": "Query Normalization",
           "range": true,
           "refId": "C"
         }
@@ -400,7 +413,7 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_cost_stats_total{cache_type=\"execution\"} [1h]))",
+          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_cost_stats_total{cache_type=\"plan\"} [1h]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -424,7 +437,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_cost_stats_total{cache_type=\"normalization\"} [1h]))",
+          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_cost_stats_total{cache_type=\"query_normalization\"} [1h]))",
           "hide": false,
           "instant": false,
           "legendFormat": "__auto",
@@ -518,7 +531,7 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_keys_stats_total{cache_type=\"execution\"} [1h]))",
+          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_keys_stats_total{cache_type=\"plan\"} [1h]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -542,7 +555,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_keys_stats_total{cache_type=\"normalization\"} [1h]))",
+          "expr": "sum by(cache_type, operation)(increase(router_graphql_cache_keys_stats_total{cache_type=\"query_normalization\"} [1h]))",
           "hide": false,
           "instant": false,
           "legendFormat": "__auto",
@@ -581,6 +594,6 @@
   "timezone": "browser",
   "title": "Router Example Dashboard",
   "uid": "be5cg59c5m6m8f",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }