diff --git a/.github/workflows/docs-staging.yml b/.github/workflows/docs-staging.yml new file mode 100644 index 00000000..d180b4d7 --- /dev/null +++ b/.github/workflows/docs-staging.yml @@ -0,0 +1,18 @@ +name: Fly Deploy Staging Docs +on: + push: + branches: + - develop + paths: + - "docs/**" +jobs: + deploy: + name: Deploy Staging Frontend + runs-on: ubuntu-latest + concurrency: deploy-group + steps: + - uses: actions/checkout@v4 + - uses: superfly/flyctl-actions/setup-flyctl@master + - run: flyctl deploy -c fly.toml --remote-only + env: + FLY_API_TOKEN: ${{ secrets.FLY_STAGING_API_TOKEN }} diff --git a/docs/Dockerfile b/docs/Dockerfile index 65fd961d..a420a10c 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,9 +1,7 @@ -FROM node:21-alpine AS build +FROM node:18 AS build ENV NEXT_TELEMETRY_DISABLED 1 - -RUN apk add --no-cache libc6-compat WORKDIR /app COPY . . RUN yarn diff --git a/docs/fly.prod.toml b/docs/fly.prod.toml index 08bac539..e2f32ae8 100644 --- a/docs/fly.prod.toml +++ b/docs/fly.prod.toml @@ -6,11 +6,8 @@ app = 'porters-docs' primary_region = 'sea' -[build] - build-target = 'production' - [http_service] - internal_port = 3005 + internal_port = 3000 auto_stop_machines = true auto_start_machines = true min_machines_running = 0 diff --git a/docs/fly.toml b/docs/fly.toml index fae1c5fd..556cb116 100644 --- a/docs/fly.toml +++ b/docs/fly.toml @@ -6,11 +6,9 @@ app = 'porters-docs-staging' primary_region = 'sea' -[build] - build-target = 'production' [http_service] - internal_port = 3005 + internal_port = 3000 auto_stop_machines = true auto_start_machines = true min_machines_running = 0 diff --git a/docs/theme.config.tsx b/docs/theme.config.tsx index c3d4c059..646d7995 100644 --- a/docs/theme.config.tsx +++ b/docs/theme.config.tsx @@ -12,7 +12,7 @@ const config: DocsThemeConfig = { docsRepositoryBase: "https://github.com/porters-xyz/gateway-demo/tree/master/docs", footer: { - text: "Open Gateway Documentation", + text: "Porters RPC Gateway Documentation", }, }; diff --git a/gateway/common/combiner.go b/gateway/common/combiner.go index 51eeac12..4b57b236 100644 --- a/gateway/common/combiner.go +++ b/gateway/common/combiner.go @@ -1,6 +1,7 @@ package common import ( + log "log/slog" ) // rather than be too chatty, combine multiple tasks into one @@ -55,6 +56,6 @@ func (c *SimpleCombiner) gen() int { return c.genVal } // Simple version only logs, need to extend to push to redis, etc func (c *SimpleCombiner) runner() func() { return func() { - //log.Printf("combined %s to %d", c.keyVal, c.intVal) + log.Debug("combining", "key", c.keyVal, "amt", c.intVal) } } diff --git a/gateway/common/config.go b/gateway/common/config.go index 80056bfa..2c811ada 100644 --- a/gateway/common/config.go +++ b/gateway/common/config.go @@ -1,8 +1,7 @@ package common import ( - "fmt" - "log" + log "log/slog" "os" "strconv" "sync" @@ -21,7 +20,7 @@ const ( REDIS_ADDR = "REDIS_ADDR" REDIS_USER = "REDIS_USER" REDIS_PASSWORD = "REDIS_PASSWORD" - + INSTRUMENT_ENABLED = "ENABLE_INSTRUMENT" ) // This may evolve to include config outside env, or use .env file for @@ -43,6 +42,7 @@ func setupConfig() *Config { config.defaults[NUM_WORKERS] = "10" config.defaults[HOST] = "localhost" config.defaults[PORT] = "9000" + config.defaults[INSTRUMENT_ENABLED] = "false" }) return config } @@ -57,7 +57,7 @@ func GetConfig(key string) string { if ok { return defaultval } else { - log.Println(fmt.Sprintf("config not set for %s, no default", key)) + log.Warn("config not set no default", "key", key) return "" } } @@ -67,8 +67,17 @@ func GetConfigInt(key string) int { configval := GetConfig(key) intval, err := strconv.Atoi(configval) if err != nil { - log.Println("Error parsing config", err) + log.Error("Error parsing config", "err", err) intval = -1 } return intval } + +func Enabled(key string) bool { + configval := GetConfig(key) + boolval, err := strconv.ParseBool(configval) + if err != nil { + boolval = false + } + return boolval +} diff --git a/gateway/common/context.go b/gateway/common/context.go index b430b0a5..9654b6d7 100644 --- a/gateway/common/context.go +++ b/gateway/common/context.go @@ -2,12 +2,21 @@ package common import ( "context" + "time" +) + +const ( + INSTRUMENT string = "INSTRUMENT_START" ) type Contextable interface { ContextKey() string } +type Instrument struct { + Timestamp time.Time +} + func UpdateContext(ctx context.Context, entity Contextable) context.Context { return context.WithValue(ctx, entity.ContextKey(), entity) } @@ -20,3 +29,13 @@ func FromContext(ctx context.Context, contextkey string) (any, bool) { return nil, false } } + +func StartInstrument() *Instrument { + return &Instrument{ + Timestamp: time.Now(), + } +} + +func (i *Instrument) ContextKey() string { + return INSTRUMENT +} diff --git a/gateway/common/prometheus.go b/gateway/common/prometheus.go index 6f442d19..d6904d82 100644 --- a/gateway/common/prometheus.go +++ b/gateway/common/prometheus.go @@ -1,6 +1,7 @@ package common import ( + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -12,6 +13,7 @@ const ( STATUS = "status" TENANT = "tenant" QUEUE = "queue" + STAGE = "stage" ) var ( @@ -23,4 +25,9 @@ var ( Name: "gateway_job_queue", Help: "If this grows too big it may effect performance, should scale up", }, []string{QUEUE}) + LatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "gateway_added_latency", + Help: "Shows how much the proxy process is adding to request", + Buckets: prometheus.ExponentialBucketsRange(float64(10 * time.Millisecond), float64(20 * time.Second), 10), + }, []string{STAGE}) ) diff --git a/gateway/common/tasks.go b/gateway/common/tasks.go index 46c1c1f0..4d8174d2 100644 --- a/gateway/common/tasks.go +++ b/gateway/common/tasks.go @@ -2,7 +2,7 @@ package common import ( "errors" - "log" + log "log/slog" "sync" "time" ) @@ -82,7 +82,7 @@ func (q *TaskQueue) CloseQueue() { return } case <-time.After(shutdownTime): - log.Println("workers not finished, work may be lost") + log.Warn("workers not finished, work may be lost") return } } @@ -111,7 +111,7 @@ func worker(q *TaskQueue) { case Runnable: task.Run() default: - log.Println("unspecified task", task, t) + log.Debug("unspecified task", "task", task, "type", t) } JobGauge.WithLabelValues("task").Set(float64(len(q.tasks))) } @@ -120,7 +120,7 @@ func worker(q *TaskQueue) { // TODO do more than log func errWorker(q *TaskQueue) { for err := range q.errors { - log.Println("error encountered", err) + log.Error("error encountered", "err", err) JobGauge.WithLabelValues("error").Dec() } } diff --git a/gateway/db/cache.go b/gateway/db/cache.go index f31015f1..a2cb2886 100644 --- a/gateway/db/cache.go +++ b/gateway/db/cache.go @@ -3,7 +3,7 @@ package db import ( "context" "fmt" - "log" + log "log/slog" "strconv" "sync" "time" @@ -49,7 +49,7 @@ func getCache() *redis.Client { // TODO figure out which redis instance to connect to opts, err := redis.ParseURL(common.GetConfig(common.REDIS_URL)) if err != nil { - log.Println(err) + log.Warn("valid REDIS_URL not provided", "err", err) opts = &redis.Options{ Addr: common.GetConfig(common.REDIS_ADDR), Username: common.GetConfig(common.REDIS_USER), @@ -58,7 +58,6 @@ func getCache() *redis.Client { } } client = redis.NewClient(opts) - log.Println("redis client:", client) }) return client } @@ -75,10 +74,8 @@ func (c *Cache) Healthcheck() *common.HealthCheckStatus { ctx := context.Background() status, err := client.Ping(ctx).Result() if err != nil { - log.Println("health error", err) hcs.AddError(REDIS, err) } else { - log.Println("health success", status) hcs.AddHealthy(REDIS, status) } @@ -143,15 +140,6 @@ func (p *Product) cache(ctx context.Context) error { return nil } -// TODO can this be done in single hop? maybe put in lua script? -// TODO need to count each use within a given scope -//func (p *productCounter) increment(ctx context.Context) { - //app := p.app - //tenant := app.tenant - // TODO increment all the right counters - // TODO decrement all the right counters -//} - func (t *Tenant) Lookup(ctx context.Context) error { fromContext, ok := common.FromContext(ctx, TENANT) if ok { @@ -161,7 +149,7 @@ func (t *Tenant) Lookup(ctx context.Context) error { result, err := getCache().HGetAll(ctx, key).Result() // TODO errors should probably cause postgres lookup if err != nil || len(result) == 0 || expired(result["cachedAt"]) { - log.Println("tenant not found", t) + log.Debug("tenant cache expired", "key", key) t.refresh(ctx) } else { t.Active, _ = strconv.ParseBool(result["active"]) @@ -178,19 +166,18 @@ func (a *App) Lookup(ctx context.Context) error { *a = *fromContext.(*App) } else { key := a.Key() - log.Println("checking cache for app", key) result, err := getCache().HGetAll(ctx, key).Result() if err != nil || len(result) == 0 || expired(result["cachedAt"]) { - log.Println("missed app", key) + log.Debug("missed app", "appkey", key) a.refresh(ctx) } else if result["missedAt"] != MISSED_FALSE { if backoff(result["missedAt"]) { - log.Println("missed and backing off") + // NOOP } else { a.refresh(ctx) } } else { - log.Println("got app", result) + log.Debug("got app from cache", "app", a.HashId()) a.Active, _ = strconv.ParseBool(result["active"]) a.Tenant.Id = result["tenant"] a.Tenant.Lookup(ctx) @@ -210,7 +197,7 @@ func (a *App) Rules(ctx context.Context) (Apprules, error) { key := iter.Val() result, err := getCache().HGetAll(ctx, key).Result() if err != nil { - log.Println("error during scan", err) + log.Error("error during scan", "err", err) continue } id := key // TODO extract id from key @@ -235,19 +222,18 @@ func (p *Product) Lookup(ctx context.Context) error { *p = *fromContext.(*Product) } else { key := p.Key() - log.Println("finding product from cache:", key) + log.Debug("finding product from cache", "prodkey", key) result, err := getCache().HGetAll(ctx, key).Result() if err != nil || len(result) == 0 || expired(result["cachedAt"]) { - log.Println("missed product", key) + log.Debug("missed product", "prodkey", key) p.refresh(ctx) } else if result["missedAt"] != MISSED_FALSE { if backoff(result["missedAt"]) { - log.Println("missed and backing off") + // NOOP } else { p.refresh(ctx) } } else { - log.Println("got product:", result) p.PoktId, _ = result["poktId"] p.Weight, _ = strconv.Atoi(result["weight"]) p.Active, _ = strconv.ParseBool(result["active"]) @@ -272,25 +258,23 @@ func RelaytxFromKey(ctx context.Context, key string) (*Relaytx, bool) { // Refresh does the psql calls to build cache func (t *Tenant) refresh(ctx context.Context) { - log.Println("refreshing tenant cache", t.Id) err := t.fetch(ctx) if err != nil { - log.Println("tenant missing, something's wrong", err) + log.Error("something's wrong", "tenant", t.Id, "err", err) // TODO how do we handle this? } else { err := t.canonicalBalance(ctx) if err != nil { - log.Println("error getting balance", err) + log.Error("error getting balance", "tenant", t.Id, "err", err) } t.cache(ctx) } } func (a *App) refresh(ctx context.Context) { - log.Println("refreshing app cache", a.Id) err := a.fetch(ctx) if err != nil { - log.Println("err seen", err) + log.Error("err seen refreshing app", "app", a.HashId(), "err", err) a.MissedAt = time.Now() } else { a.Tenant.Lookup(ctx) @@ -299,7 +283,7 @@ func (a *App) refresh(ctx context.Context) { rules, err := a.fetchRules(ctx) if err != nil { - log.Println("error accessing rules", err) + log.Error("error accessing rules", "app", a.HashId(), "err", err) return } for _, r := range rules { @@ -308,10 +292,9 @@ func (a *App) refresh(ctx context.Context) { } func (p *Product) refresh(ctx context.Context) { - log.Println("refreshing product cache", p.Id) err := p.fetch(ctx) if err != nil { - log.Println("err getting product", err) + log.Error("err getting product", "product", p.Name, "err", err) p.MissedAt = time.Now() } p.cache(ctx) @@ -416,7 +399,7 @@ func ReconcileRelays(ctx context.Context, rtx *Relaytx) (func() bool, error) { background := context.Background() pgerr := rtx.write(background) if pgerr != nil { - log.Println("couldn't write relaytx", pgerr) + log.Error("couldn't write relaytx", "pgerr", pgerr) return false } return true diff --git a/gateway/db/canonical.go b/gateway/db/canonical.go index 8c1195ee..b1b4f8cc 100644 --- a/gateway/db/canonical.go +++ b/gateway/db/canonical.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" "errors" - "log" + log "log/slog" "sync" "github.com/lib/pq" @@ -36,7 +36,7 @@ func getCanonicalDB() *sql.DB { connector, err := pq.NewConnector(connStr) if err != nil { // TODO handle nicely, maybe retry? - log.Fatal(err) + log.Error("Cannot connect to postgres", "err", err) } postgresPool = sql.OpenDB(connector) }) diff --git a/gateway/fly.prod.toml b/gateway/fly.prod.toml index 851606ed..92cbe899 100644 --- a/gateway/fly.prod.toml +++ b/gateway/fly.prod.toml @@ -21,10 +21,15 @@ primary_region = 'sea' [http_service] internal_port = 9000 force_https = true - auto_stop_machines = true + auto_stop_machines = false auto_start_machines = true - min_machines_running = 1 processes = ['app'] + [[http_service.checks]] + grace_period = '2s' + interval = '30s' + method = 'GET' + timeout = '3s' + path = '/health' [[vm]] memory = '1gb' diff --git a/gateway/fly.toml b/gateway/fly.toml index 999b89fc..8a23a692 100644 --- a/gateway/fly.toml +++ b/gateway/fly.toml @@ -25,6 +25,12 @@ primary_region = 'sea' auto_start_machines = true min_machines_running = 1 processes = ['app'] + [[http_service.checks]] + grace_period = '2s' + interval = '30s' + method = 'GET' + timeout = '3s' + path = '/health' [[vm]] memory = '1gb' diff --git a/gateway/gateway.go b/gateway/gateway.go index ad608a7d..9b7d3455 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -1,7 +1,7 @@ package main import ( - "log" + log "log/slog" "os" "os/signal" "sync" @@ -15,7 +15,7 @@ func gateway() { // Start job queue common.GetTaskQueue().SetupWorkers() - log.Println("starting gateway") + log.Info("starting gateway") proxy.Start() done := make(chan os.Signal, 1) diff --git a/gateway/go.mod b/gateway/go.mod index 8eb9b60b..316fc774 100644 --- a/gateway/go.mod +++ b/gateway/go.mod @@ -1,6 +1,6 @@ module porters -go 1.21 +go 1.21.9 require ( github.com/go-redis/redis_rate/v10 v10.0.1 diff --git a/gateway/justfile b/gateway/justfile index 74f6cf0b..3d624dec 100644 --- a/gateway/justfile +++ b/gateway/justfile @@ -16,6 +16,12 @@ docker-build: docker-run: docker-build docker run -d -p 9999:9999 porters +prod-status: + fly status -c fly.prod.toml + +prod-deploy: + fly scale count 3 --region sea,sin,ams -c fly.prod.toml + # I was typing `just status` a lot so put this here # maybe overload with something else status: diff --git a/gateway/plugins/apikeyauth.go b/gateway/plugins/apikeyauth.go index 83c68490..94d49ec5 100644 --- a/gateway/plugins/apikeyauth.go +++ b/gateway/plugins/apikeyauth.go @@ -5,7 +5,7 @@ package plugins import ( "context" - "log" + log "log/slog" "net/http" "porters/common" @@ -24,7 +24,7 @@ func (a *ApiKeyAuth) Name() string { func (a *ApiKeyAuth) Load() { // load plugin - log.Println("loading", a.Name()) + log.Debug("loading plugin", "plugin", a.Name()) } func (a *ApiKeyAuth) Key() string { @@ -70,7 +70,7 @@ func (a *ApiKeyAuth) getRulesForScope(ctx context.Context, app *db.App) []string apirules := make([]string, 0) rules, err := app.Rules(ctx) if err != nil { - log.Println("error getting rules", err) + log.Error("error getting rules", "app", app.HashId(), "err", err) } else { for _, rule := range rules { if rule.RuleType != "secret-key" || !rule.Active { diff --git a/gateway/plugins/balance.go b/gateway/plugins/balance.go index 438403ed..692ca5a9 100644 --- a/gateway/plugins/balance.go +++ b/gateway/plugins/balance.go @@ -5,7 +5,7 @@ package plugins import ( "context" "fmt" - "log" + log "log/slog" "net/http" "porters/common" @@ -38,7 +38,7 @@ func (b *BalanceTracker) Key() string { func (b *BalanceTracker) Load() { // Setup any plugin state - log.Println("Loading", b.Name()) + log.Debug("Loading plugin", "plugin", b.Name()) } // TODO optim: script this to avoid multi-hops @@ -47,7 +47,6 @@ func (b *BalanceTracker) HandleRequest(req *http.Request) error { appId := proxy.PluckAppId(req) app := &db.App{Id: appId} err := app.Lookup(ctx) - log.Println("app:", app) if err != nil { return proxy.NewHTTPError(http.StatusNotFound) } @@ -63,12 +62,11 @@ func (b *BalanceTracker) HandleRequest(req *http.Request) error { ctx = common.UpdateContext(ctx, app) // TODO Check that balance is greater than or equal to req weight if bal.cachedBalance > 0 { - log.Println("balance remaining") lifecycle := proxy.SetStageComplete(ctx, proxy.BalanceCheck|proxy.AccountLookup) ctx = common.UpdateContext(ctx, lifecycle) *req = *req.WithContext(ctx) } else { - log.Println("none remaining", appId) + log.Debug("no balance remaining", "app", app.HashId()) return proxy.BalanceExceededError } return nil diff --git a/gateway/plugins/blocker.go b/gateway/plugins/blocker.go index 37934384..4db9e252 100644 --- a/gateway/plugins/blocker.go +++ b/gateway/plugins/blocker.go @@ -3,14 +3,14 @@ package plugins import ( "errors" "fmt" - "log" + log "log/slog" "net/http" ) type Blocker struct {} func (b Blocker) Load() { - log.Println("loading " + b.Name()) + log.Debug("loading plugin", "plugin", b.Name()) } func (b Blocker) Name() string { @@ -22,11 +22,11 @@ func (b Blocker) Key() string { } func (b Blocker) HandleRequest(req *http.Request) error { - log.Println("logging block") + log.Debug("logging block, used for testing") return errors.New(fmt.Sprint("blocked by prehandler", b.Name())) } func (b Blocker) HandleResponse(resp *http.Response) error { - log.Println("logging block (post)") + log.Debug("logging block after proxy, used for testing") return errors.New(fmt.Sprint("blocked by posthandler", b.Name())) } diff --git a/gateway/plugins/counter.go b/gateway/plugins/counter.go index ccdb8cb4..e763a2e9 100644 --- a/gateway/plugins/counter.go +++ b/gateway/plugins/counter.go @@ -1,9 +1,8 @@ package plugins import ( - "log" + log "log/slog" "net/http" - "strconv" "porters/db" ) @@ -11,7 +10,7 @@ import ( type Counter struct {} func (c Counter) Load() { - log.Println("loading " + c.Name()) + log.Debug("loading plugin", "plugin", c.Name()) } func (c Counter) Name() string { @@ -31,7 +30,6 @@ func (c Counter) Field() string { // TODO make this asynchronous and remove header set func (c Counter) HandleResponse(resp *http.Response) error { newCount := db.IncrementCounter(resp.Request.Context(), c.Key(), 1) - log.Println("count", newCount) - resp.Header.Set("X-Counter", strconv.Itoa(newCount)) + log.Debug("counting request", "count", newCount) return nil } diff --git a/gateway/plugins/leaky.go b/gateway/plugins/leaky.go index 584ca5c6..132ede33 100644 --- a/gateway/plugins/leaky.go +++ b/gateway/plugins/leaky.go @@ -5,7 +5,7 @@ package plugins import ( "context" "fmt" - "log" + log "log/slog" "net/http" rl "github.com/go-redis/redis_rate/v10" @@ -39,7 +39,7 @@ func (l *LeakyBucketPlugin) HandleRequest(req *http.Request) error { app := &db.App{Id: appId} err := app.Lookup(ctx) if err != nil { - log.Println("err", err) + log.Error("unable to lookup app", "app", app.HashId(), "err", err) } buckets := l.getBucketsForScope(ctx, app) limiter := db.Limiter() @@ -50,7 +50,7 @@ func (l *LeakyBucketPlugin) HandleRequest(req *http.Request) error { return proxy.NewHTTPError(http.StatusBadGateway) } - log.Println("rate limit result:", res) + log.Debug("rate limit result", "allowed", res.Allowed) // rate limited if res.Allowed == 0 { @@ -69,7 +69,7 @@ func (l *LeakyBucketPlugin) getBucketsForScope(ctx context.Context, app *db.App) rules, err := app.Rules(ctx) if err != nil { // TODO should this stop all proxying? - log.Println("error getting rules", err) + log.Error("error getting rules", "app", app.HashId(), "err", err) return buckets } for _, rule := range rules { @@ -78,7 +78,7 @@ func (l *LeakyBucketPlugin) getBucketsForScope(ctx context.Context, app *db.App) } rate, err := utils.ParseRate(rule.Value) if err != nil { - log.Println("Invalid rate") + log.Error("Invalid rate found", "rule", rule.Value, "err", err) continue } diff --git a/gateway/plugins/noop.go b/gateway/plugins/noop.go deleted file mode 100644 index da869f3c..00000000 --- a/gateway/plugins/noop.go +++ /dev/null @@ -1,21 +0,0 @@ -package plugins - -import ( - "log" -) - -// Plugins that don't implement pre or post handler exist outside of request -// lifecycle -type Noop struct {} - -func (n Noop) Load() { - log.Println("loading " + n.Name()) -} - -func (n Noop) Name() string { - return "noop" -} - -func (n Noop) Key() string { - return "NOOP" -} diff --git a/gateway/plugins/noopfilter.go b/gateway/plugins/noopfilter.go index 885700e4..ab0b0819 100644 --- a/gateway/plugins/noopfilter.go +++ b/gateway/plugins/noopfilter.go @@ -1,7 +1,7 @@ package plugins import ( - "log" + log "log/slog" "net/http" "porters/common" @@ -14,7 +14,7 @@ type NoopFilter struct { } func (n NoopFilter) Load() { - log.Println("loading " + n.Name()) + log.Debug("loading plugin", "plugin", n.Name()) } func (n NoopFilter) Name() string { diff --git a/gateway/plugins/origin.go b/gateway/plugins/origin.go index b454d818..17a2404d 100644 --- a/gateway/plugins/origin.go +++ b/gateway/plugins/origin.go @@ -2,7 +2,7 @@ package plugins import ( "context" - "log" + log "log/slog" "net/http" "regexp" @@ -28,7 +28,7 @@ func (a *AllowedOriginFilter) Key() string { } func (a *AllowedOriginFilter) Load() { - log.Println("loading", a.Name()) + log.Debug("loading plugin", "plugin", a.Name()) } func (a *AllowedOriginFilter) HandleRequest(req *http.Request) error { @@ -63,7 +63,7 @@ func (a *AllowedOriginFilter) getRulesForScope(ctx context.Context, app *db.App) origins := make([]regexp.Regexp, 0) rules, err := app.Rules(ctx) if err != nil { - log.Println("couldn't get rules", err) + log.Error("couldn't get rules", "app", app.HashId(), "err", err) } else { for _, rule := range rules { if rule.RuleType != ALLOWED_ORIGIN || !rule.Active { @@ -71,7 +71,7 @@ func (a *AllowedOriginFilter) getRulesForScope(ctx context.Context, app *db.App) } matcher, err := regexp.Compile(rule.Value) if err != nil { - log.Println("error compiling origin regex", err) + log.Error("error compiling origin regex", "regex", rule.Value, "err", err) continue } origins = append(origins, *matcher) diff --git a/gateway/plugins/productfilter.go b/gateway/plugins/productfilter.go index 2e83521d..336b05c8 100644 --- a/gateway/plugins/productfilter.go +++ b/gateway/plugins/productfilter.go @@ -2,7 +2,7 @@ package plugins import ( "context" - "log" + log "log/slog" "net/http" "porters/db" @@ -26,7 +26,7 @@ func (p *ProductFilter) Key() string { } func (p *ProductFilter) Load() { - log.Println("loading", p.Name()) + log.Debug("loading plugin", "plugin", p.Name()) } func (p *ProductFilter) HandleRequest(req *http.Request) error { @@ -60,13 +60,13 @@ func (p *ProductFilter) getRulesForScope(ctx context.Context, app *db.App) []str products := make([]string, 0) rules, err := app.Rules(ctx) if err != nil { - log.Println("couldn't read rules", err) + log.Error("couldn't read rules", "app", app.HashId(), "err", err) } else { for _, rule := range rules { if rule.RuleType != ALLOWED_PRODUCTS || !rule.Active { continue } - log.Println("allowing", rule.Value) + log.Debug("allowing product", "product", rule.Value) products = append(products, rule.Value) } } diff --git a/gateway/plugins/useragent.go b/gateway/plugins/useragent.go index 46205726..0aa6a24b 100644 --- a/gateway/plugins/useragent.go +++ b/gateway/plugins/useragent.go @@ -2,7 +2,7 @@ package plugins import ( "context" - "log" + log "log/slog" "net/http" "regexp" @@ -27,7 +27,7 @@ func (u *UserAgentFilter) Key() string { } func (u *UserAgentFilter) Load() { - log.Println("Loading", u.Name()) + log.Debug("Loading plugin", "plugin", u.Name()) } func (u *UserAgentFilter) HandleRequest(req *http.Request) error { @@ -60,7 +60,7 @@ func (u *UserAgentFilter) getRulesForScope(ctx context.Context, app *db.App) []r useragents := make([]regexp.Regexp, 0) rules, err := app.Rules(ctx) if err != nil { - log.Println("couldn't get rules", err) + log.Error("couldn't get rules", "err", err) } else { for _, rule := range rules { if rule.RuleType != UA_TYPE_ID || !rule.Active { @@ -68,7 +68,7 @@ func (u *UserAgentFilter) getRulesForScope(ctx context.Context, app *db.App) []r } matcher, err := regexp.Compile(rule.Value) if err != nil { - log.Println("unable to compile regexp", err) + log.Error("unable to compile regexp", "regex", rule.Value, "err", err) } else { useragents = append(useragents, *matcher) } diff --git a/gateway/proxy/httpinstr.go b/gateway/proxy/httpinstr.go deleted file mode 100644 index 6c242ed3..00000000 --- a/gateway/proxy/httpinstr.go +++ /dev/null @@ -1,40 +0,0 @@ -package proxy - -import ( - "io" - "net/http" -) - -// Instruments http requests to allow for handling details of http -// Might not be needed, but useful if middleware is still required somewhere -type ResponseRecorder struct { - http.ResponseWriter - Status int - Tee io.Writer -} - -// nil for tee is fine if not wanting to capture output (likely only useful for -// debugging) -func WithRecorder(h http.Handler, tee io.Writer) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - recorder := &ResponseRecorder{ - ResponseWriter: w, - Status: 200, - Tee: tee, - } - h.ServeHTTP(recorder, r) - - }) -} - -func (r *ResponseRecorder) WriteHeader(status int) { - r.Status = status - r.ResponseWriter.WriteHeader(status) -} - -func (r *ResponseRecorder) Write(bytes []byte) (int, error) { - if r.Tee != nil { - r.Tee.Write(bytes) - } - return r.ResponseWriter.Write(bytes) -} diff --git a/gateway/proxy/proxy.go b/gateway/proxy/proxy.go index 51a42ab2..ffc845c8 100644 --- a/gateway/proxy/proxy.go +++ b/gateway/proxy/proxy.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "log" + log "log/slog" "net/url" "net/http" "net/http/httputil" @@ -23,15 +23,14 @@ func Start() { // TODO grab url for gateway kit proxyUrl := common.GetConfig(common.PROXY_TO) remote, err := url.Parse(proxyUrl) - log.Println("remote", remote) if err != nil { - // TODO probably panic here, can't proxy anywhere - log.Println(err) + log.Error("unable to parse proxy to", "err", err) + panic("unable to start with invalid remote url") } + log.Debug("proxying to remote", "url", remote) handler := func(proxy *httputil.ReverseProxy) func(http.ResponseWriter, *http.Request) { return func(resp http.ResponseWriter, req *http.Request) { - log.Println(req.URL) setupContext(req) proxy.ServeHTTP(resp, req) } @@ -51,7 +50,7 @@ func Start() { go func() { err := server.ListenAndServe() if err != nil { - log.Println("server error", err) + log.Error("server error encountered", "err", err) } }() } @@ -64,9 +63,9 @@ func Stop() { err := server.Shutdown(ctx) if err != nil { - log.Println("error shutting down", err) + log.Error("error shutting down", "err", err) } else { - log.Println("shutdown successful") + log.Info("shutdown successful") } } @@ -89,14 +88,12 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy { poktId := lookupPoktId(req) target := utils.NewTarget(remote, poktId) req.URL = target.URL() - log.Println("proxying to", target.URL()) cancel := RequestCanceler(req) for _, p := range (*reg).plugins { h, ok := p.(PreHandler) if ok { - log.Println("encountered", p.Name()) select { case <-req.Context().Done(): return @@ -113,13 +110,36 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy { lifecycle := lifecycleFromContext(req.Context()) if !lifecycle.checkComplete() { err := LifecycleIncompleteError - log.Println("lifecycle incomplete", lifecycle) + log.Debug("lifecycle incomplete", "mask", lifecycle) cancel(err) } + + if common.Enabled(common.INSTRUMENT_ENABLED) { + ctx := req.Context() + instr, ok := common.FromContext(ctx, common.INSTRUMENT) + if ok { + start := instr.(*common.Instrument).Timestamp + elapsed := time.Now().Sub(start) + common.LatencyHistogram.WithLabelValues("setup").Observe(float64(elapsed)) + + ctx = common.UpdateContext(ctx, common.StartInstrument()) + *req = *req.WithContext(ctx) + } + } } revProxy.ModifyResponse = func(resp *http.Response) error { ctx := resp.Request.Context() + + if common.Enabled(common.INSTRUMENT_ENABLED) { + instr, ok := common.FromContext(ctx, common.INSTRUMENT) + if ok { + start := instr.(*common.Instrument).Timestamp + elapsed := time.Now().Sub(start) + common.LatencyHistogram.WithLabelValues("serve").Observe(float64(elapsed)) + } + } + var err error for _, p := range (*reg).plugins { h, ok := p.(PostHandler) @@ -136,7 +156,6 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy { common.GetTaskQueue().Add(updater) } - log.Println("returning", err) return err } @@ -149,7 +168,7 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy { updater := db.NewUsageUpdater(ctx, "failure") common.GetTaskQueue().Add(updater) - log.Println("cancel cause", cause) + log.Debug("cancel cause", "cause", cause) if errors.As(cause, &httpErr) { status := httpErr.code http.Error(resp, http.StatusText(status), status) @@ -165,8 +184,11 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy { func setupContext(req *http.Request) { // TODO read ctx from request and make any modifications ctx := req.Context() - lifecyclectx := common.UpdateContext(ctx, &Lifecycle{}) - *req = *req.WithContext(lifecyclectx) + ctx = common.UpdateContext(ctx, &Lifecycle{}) + if common.Enabled(common.INSTRUMENT_ENABLED) { + ctx = common.UpdateContext(ctx, common.StartInstrument()) + } + *req = *req.WithContext(ctx) } func lookupPoktId(req *http.Request) string { @@ -176,7 +198,7 @@ func lookupPoktId(req *http.Request) string { err := product.Lookup(ctx) if err != nil { // TODO pick appropriate HTTP code - log.Println("product not found", err) + log.Error("product not found", "product", product.Name, "err", err) } productCtx := common.UpdateContext(ctx, product) *req = *req.WithContext(productCtx) diff --git a/gateway/proxy/reconciler.go b/gateway/proxy/reconciler.go index 964eebae..e7889e92 100644 --- a/gateway/proxy/reconciler.go +++ b/gateway/proxy/reconciler.go @@ -2,7 +2,7 @@ package proxy import ( "context" - "log" + log "log/slog" "time" "porters/common" @@ -68,7 +68,7 @@ func (t *reconcileTask) Run() { ctx := context.Background() replayfunc, err := db.ReconcileRelays(ctx, t.relaytx) if err != nil { - log.Println("unable to access cached relay use", err) + log.Error("unable to access cached relay use", "err", err) } t.RetryTask = common.NewRetryTask(replayfunc, 5, 1 * time.Minute) diff --git a/gateway/proxy/registry.go b/gateway/proxy/registry.go index 260d2c95..2ca43185 100644 --- a/gateway/proxy/registry.go +++ b/gateway/proxy/registry.go @@ -5,7 +5,7 @@ package proxy import ( "errors" - "log" + log "log/slog" "sync" ) @@ -31,7 +31,7 @@ func Register(plugin Plugin) { _ = GetRegistry() // init singleton err := avoidCollision(plugin) if err != nil { - log.Println("unable to load plugin", plugin.Name(), "due to", err.Error()) + log.Error("unable to load plugin", "plugin", plugin.Name(), "err", err.Error()) return } plugin.Load() diff --git a/justfile b/justfile index 0c539a56..11b50b5f 100644 --- a/justfile +++ b/justfile @@ -24,3 +24,8 @@ build-frontend: cd ./web-portal/frontend && pnpm install && pnpm build serve-frontend: cd ./web-portal/frontend && pnpm install && pnpm start + +deploy-prod: + @just gateway/prod-deploy + @just services/gatewaykit/prod-deploy + @just services/redis/prod-deploy diff --git a/services/gatewaykit/fly.prod.toml b/services/gatewaykit/fly.prod.toml index ed710bcf..91e6ed4e 100644 --- a/services/gatewaykit/fly.prod.toml +++ b/services/gatewaykit/fly.prod.toml @@ -9,6 +9,9 @@ primary_region = 'sea' [build] image = 'ghcr.io/pokt-network/pocket-gateway-server:0.3.0' +[deploy] + strategy = 'canary' + [env] ALTRUIST_REQUEST_TIMEOUT = '10s' ENVIRONMENT_STAGE = 'production' @@ -21,17 +24,24 @@ primary_region = 'sea' [[services]] internal_port = 8080 protocol = 'tcp' - auto_stop_machines = true + auto_stop_machines = false auto_start_machines = true - min_machines_running = 1 [[services.ports]] handlers = ["http"] port = 8080 force_https = false + [[services.http_checks]] + interval = 5000 + grace_period = '10s' + method = 'get' + path = '/metrics' + protocol = 'http' + timeout = 1000 + [[vm]] - memory = '1gb' + memory = '4gb' cpu_kind = 'shared' - cpus = 1 + cpus = 2 [[metrics]] port = 8080 diff --git a/services/gatewaykit/justfile b/services/gatewaykit/justfile new file mode 100644 index 00000000..0f340779 --- /dev/null +++ b/services/gatewaykit/justfile @@ -0,0 +1,8 @@ +default: + @just --list + +prod-status: + fly status -c fly.prod.toml + +prod-deploy: + fly scale count 3 --region sea,sin,ams -c fly.prod.toml diff --git a/services/postgres/fly.toml b/services/postgres/fly.toml new file mode 100644 index 00000000..06a9aed4 --- /dev/null +++ b/services/postgres/fly.toml @@ -0,0 +1,3 @@ +app = 'porters-schema' + + diff --git a/services/redis/fly.prod.toml b/services/redis/fly.prod.toml index f1e55219..e5b344dd 100644 --- a/services/redis/fly.prod.toml +++ b/services/redis/fly.prod.toml @@ -1,4 +1,5 @@ app = 'porters-redis' +primary_region = 'sea' [mounts] destination = "/data" diff --git a/services/redis/justfile b/services/redis/justfile new file mode 100644 index 00000000..8de04e31 --- /dev/null +++ b/services/redis/justfile @@ -0,0 +1,8 @@ +default: + @just --list + +prod-status: + fly status -c fly.prod.toml + +prod-deploy: + fly scale count 3 --region sea,sin,ams --max-per-region 1 -c fly.prod.toml diff --git a/web-portal/backend/Dockerfile.fly b/web-portal/backend/Dockerfile.fly index a8581a5f..004e54e9 100644 --- a/web-portal/backend/Dockerfile.fly +++ b/web-portal/backend/Dockerfile.fly @@ -1,5 +1,6 @@ # This file is used to specifically build portal backend image -FROM registry.fly.io/porters-schema As build +ARG SCHEMA_VERSION=latest +FROM registry.fly.io/porters-schema:${SCHEMA_VERSION} As build WORKDIR /usr/src/app diff --git a/web-portal/backend/fly.prod.toml b/web-portal/backend/fly.prod.toml index 35e6a40e..c6557115 100644 --- a/web-portal/backend/fly.prod.toml +++ b/web-portal/backend/fly.prod.toml @@ -4,20 +4,27 @@ primary_region = 'sea' [build] dockerfile = 'Dockerfile.fly' +[build.args] + SCHEMA_VERSION = '0.0.1' + [env] ### SECRETS ### # DATABASE_URL = ... # ONEINCH_API_KEY = ... # OX_API_KEY = ... # RPC_KEY = ... + PROM_URL = 'https://api.fly.io/prometheus/porters/api/v1/' -[http_service] +[[services]] internal_port = 4000 - force_https = false + protocol = 'tcp' auto_stop_machines = true auto_start_machines = true min_machines_running = 0 - processes = ['app'] + [[services.ports]] + handlers = ["http"] + port = 4000 + force_https = false [[vm]] memory = '1gb' diff --git a/web-portal/backend/fly.toml b/web-portal/backend/fly.toml index aef53708..d80a8bea 100644 --- a/web-portal/backend/fly.toml +++ b/web-portal/backend/fly.toml @@ -15,6 +15,7 @@ primary_region = 'sea' # ONEINCH_API_KEY = ... # OX_API_KEY = ... # RPC_KEY = ... + PROM_URL = 'https://api.fly.io/prometheus/porters-staging/api/v1/' [http_service] internal_port = 4000 diff --git a/web-portal/backend/package.json b/web-portal/backend/package.json index 21bcb5b0..2fc96a08 100644 --- a/web-portal/backend/package.json +++ b/web-portal/backend/package.json @@ -30,10 +30,12 @@ "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", "@nestjs/schedule": "^4.0.2", + "@nestjs/swagger": "^7.3.1", "@prisma/client": "^5.9.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cookie-parser": "^1.4.6", + "date-fns": "^3.6.0", "iron-session": "^8.0.1", "nestjs-prisma": "^0.22.0", "reflect-metadata": "^0.1.13", diff --git a/web-portal/backend/src/app.module.ts b/web-portal/backend/src/app.module.ts index 522626f8..65fe4023 100644 --- a/web-portal/backend/src/app.module.ts +++ b/web-portal/backend/src/app.module.ts @@ -1,5 +1,5 @@ +import { APP_GUARD } from '@nestjs/core' import { Module } from '@nestjs/common'; -import { AppService } from './app.service'; import { ConfigModule } from '@nestjs/config'; import { CustomPrismaModule } from 'nestjs-prisma'; import { PrismaClient } from '@/.generated/client'; @@ -10,6 +10,10 @@ import { UserModule } from './user/user.module'; import { AppsModule } from './apps/apps.module'; import { OrgModule } from './org/org.module'; import { UtilsModule } from './utils/utils.module'; +import { UsageModule } from './usage/usage.module'; +import { SiweService } from './siwe/siwe.service'; +import { AuthGuard } from './guards/auth.guard'; +import { AppsService } from './apps/apps.service'; @Module({ imports: [ @@ -27,8 +31,16 @@ import { UtilsModule } from './utils/utils.module'; AppsModule, OrgModule, UtilsModule, + UsageModule, + ], + providers: [ + SiweService, + AppsService, + { + provide: APP_GUARD, + useClass: AuthGuard, + } ], - providers: [AppService], controllers: [AppController], }) -export class AppModule {} +export class AppModule { } diff --git a/web-portal/backend/src/app.service.ts b/web-portal/backend/src/app.service.ts deleted file mode 100644 index 3f88f1b6..00000000 --- a/web-portal/backend/src/app.service.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Injectable, Inject } from '@nestjs/common'; -import { CustomPrismaService } from 'nestjs-prisma'; -import { PrismaClient } from '@/.generated/client'; - -@Injectable() -export class AppService { - constructor( - @Inject('Postgres') - private prisma: CustomPrismaService, // <-- Inject the PrismaClient - ) {} - - getHello(): string { - return this.prisma.client.tenant.findMany().toString(); - } -} diff --git a/web-portal/backend/src/apps/apps.controller.ts b/web-portal/backend/src/apps/apps.controller.ts index 045b117b..0cdefa93 100644 --- a/web-portal/backend/src/apps/apps.controller.ts +++ b/web-portal/backend/src/apps/apps.controller.ts @@ -2,15 +2,16 @@ import { Controller, Get, Param, - UseGuards, Body, Post, Put, Patch, + Req, } from '@nestjs/common'; +import { Request } from 'express'; import { AppsService } from './apps.service'; -import { AuthGuard } from '../guards/auth.guard'; import { Delete } from '@nestjs/common'; +import { UserService } from '../user/user.service'; // TODO: create a centralised interface file? interface CreateAppDto { @@ -19,34 +20,39 @@ interface CreateAppDto { } @Controller('apps') -@UseGuards(AuthGuard) export class AppsController { - constructor(private readonly appsService: AppsService) {} + constructor( + private readonly appsService: AppsService, + private readonly userService: UserService) { } - @Get(':userAddress') - async getUserApps(@Param('userAddress') userAddress: string) { - // @note: This action fetches apps by userAddress; + @Get() + async getUserApps(@Req() req: Request) { + // @note: This action fetches apps by userAddress retrieved from cookie; + const userAddress = await this.userService.getUserAddress(req); return this.appsService.getAppsByUser(userAddress); } - @Post(':userAddress') + @Post() async createApp( - @Param('userAddress') userAddress: string, + @Req() req: Request, @Body() createAppDto: CreateAppDto, ) { - // @note: This action creates app for given userAddress; + // @note: This action creates app for retrived userAddress from cookie; + const userAddress = await this.userService.getUserAddress(req); const { name, description } = createAppDto; return this.appsService.createApp(userAddress, name, description); } @Delete(':appId') - async deleteApp(@Param('appId') appId: string) { + async deleteApp( + @Param('appId') appId: string) { // @note: This action deletes app for given appId; return this.appsService.deleteApp(appId); } @Patch(':appId') - async updateApp(@Param('appId') appId: string, @Body() updateAppDto: any) { + async updateApp( + @Param('appId') appId: string, @Body() updateAppDto: any) { // @note: This action updates app for given appId; return this.appsService.updateApp(appId, updateAppDto); } @@ -56,50 +62,52 @@ export class AppsController { @Param('appId') appId: string, @Body() createAppRuleDto: { - ruleId: string; + ruleName: string; data: string[]; }, ) { // @note: This action creates app rule given appId; - return this.appsService.createAppRule(appId, createAppRuleDto); + return this.appsService.createAppRule(appId, createAppRuleDto.ruleName, createAppRuleDto.data); } - @Patch(':appId/rule/:ruleId') + @Patch(':appId/rule/:ruleName') async updateAppRule( @Param('appId') appId: string, - @Param('ruleId') ruleId: string, + @Param('ruleName') ruleName: string, @Body() updateAppRuleDto: string[], ) { - // @note: This action updates app rule for given appId and ruleId; - return this.appsService.updateAppRule(appId, ruleId, updateAppRuleDto); + // @note: This action updates app rule for given appId and ruleName; + return this.appsService.updateAppRule(appId, ruleName, updateAppRuleDto); } - @Delete(':appId/rule/:ruleId') + @Delete(':appId/rule/:ruleName') async deleteAppRule( @Param('appId') appId: string, - @Param('ruleId') ruleId: string, + @Param('ruleName') ruleName: string, ) { - // @note: This action deletes app rule for given appId and ruleId; - return this.appsService.deleteAppRule(appId, ruleId); + // @note: This action deletes app rule for given appId and ruleName; + return this.appsService.deleteAppRule(appId, ruleName); } @Patch(':appId/rules') async batchUpdateAppRules( @Param('appId') appId: string, - @Body() updateRulesDto: { ruleId: string; data: string[] }[], + @Body() updateRulesDto: { ruleName: string; data: string[] }, ) { // @note: This action updates app rules in bulk for given appId; - return this.appsService.batchUpdateAppRules(appId, updateRulesDto); + return this.appsService.batchUpdateAppRules(appId, updateRulesDto.ruleName, updateRulesDto.data); } @Put(':appId/secret') - async updateAppSecret(@Param('appId') appId: string) { + async updateAppSecret( + @Param('appId') appId: string) { // @note: This action updates app secret for given appId; return this.appsService.updateSecretKeyRule(appId, 'generate'); } @Delete(':appId/secret') - async deleteAppSecret(@Param('appId') appId: string) { + async deleteAppSecret( + @Param('appId') appId: string) { // @note: This action deletes app secret for given appId; return this.appsService.updateSecretKeyRule(appId, 'delete'); } diff --git a/web-portal/backend/src/apps/apps.service.ts b/web-portal/backend/src/apps/apps.service.ts index 27f3c5e7..cc9268db 100644 --- a/web-portal/backend/src/apps/apps.service.ts +++ b/web-portal/backend/src/apps/apps.service.ts @@ -1,307 +1,359 @@ import { Injectable, Inject, HttpException, HttpStatus } from '@nestjs/common'; import { CustomPrismaService } from 'nestjs-prisma'; -import { AppRule, PrismaClient } from '@/.generated/client'; +import { AppRule, PrismaClient, Tenant, RuleType } from '@/.generated/client'; import { UserService } from '../user/user.service'; import { createHash, randomBytes } from 'crypto'; +import { Request } from 'express'; @Injectable() export class AppsService { - constructor( - @Inject('Postgres') - private prisma: CustomPrismaService, // <-- Inject the PrismaClient - private userService: UserService, - ) { } - - async getTenantsByUser(userAddress: string) { - const user = await this.userService.getOrCreate(userAddress); - - const enterprises = user.orgs.map((org) => org.enterpriseId); - - const tenants = await this.prisma.client.tenant.findMany({ - where: { - enterpriseId: { - in: enterprises, - }, - deletedAt: null, - }, - }); - - if (!tenants || tenants.length === 0) { - throw new HttpException('No tenants found', HttpStatus.NOT_FOUND); - } - return tenants; + constructor( + @Inject('Postgres') + private prisma: CustomPrismaService, // <-- Inject the PrismaClient + private userService: UserService, + ) { } + + async getRuleType(ruleName: string) { + const ruleType = await this.prisma.client.ruleType.findFirst({ + where: { name: ruleName, deletedAt: null }, + }); + + if (!ruleType) { + throw new HttpException(`Trying to get invalid ruleType`, HttpStatus.BAD_REQUEST) } - async getAppsByUser(userAddress: string) { - const tenants = await this.getTenantsByUser(userAddress); - const apps = await this.prisma.client.app.findMany({ - where: { - tenantId: { - in: tenants.map((tenant) => tenant.id), - }, - deletedAt: null, - }, - include: { - appRules: { - where: { - deletedAt: null, - } - } - }, - }); - - if (!apps || apps?.length === 0) { - throw new HttpException('No apps found', HttpStatus.NOT_FOUND); + return ruleType as RuleType; + } + + async getTenantsByUser(userAddress: string) { + + const user = await this.userService.getOrCreate(userAddress); + + const enterprises = user.orgs.map((org) => org.enterpriseId); + + const tenants = await this.prisma.client.tenant.findMany({ + where: { + enterpriseId: { + in: enterprises, + }, + deletedAt: null, + }, + }); + + if (!tenants || tenants.length === 0) { + throw new HttpException('No tenants found', HttpStatus.NOT_FOUND); + } + return tenants; + } + + async getAppsByUser(userAddress: string) { + const tenants = await this.getTenantsByUser(userAddress); + + const tenantIds = tenants.map((tenant: Tenant) => tenant.id) + const apps = await this.prisma.client.app.findMany({ + where: { + tenantId: { + in: tenantIds, + }, + deletedAt: null, + }, + include: { + appRules: { + where: { + deletedAt: null + } } - return apps; + }, + }); + + if (!apps || apps?.length === 0) { + throw new HttpException('No apps found', HttpStatus.NOT_FOUND); } - async createApp( - userAddress: string, - name: string, - description: string | null | undefined, - ) { - const tenants = await this.getTenantsByUser(userAddress); - - if (!tenants) return; - const newApp = await this.prisma.client.app.create({ - data: { - tenantId: tenants[0].id, - name, - description, - }, - }); - - if (!newApp) { - return new HttpException( - `Could not create app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } + return apps; + } + + async createApp( + userAddress: string, + name: string, + description: string | null | undefined, + ) { + const tenants = await this.getTenantsByUser(userAddress); + + if (!tenants) return; + const newApp = await this.prisma.client.app.create({ + data: { + tenantId: tenants[0].id, + name, + description, + }, + }); + + if (!newApp) { + return new HttpException( + `Could not create app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + + return newApp; + } - return newApp; + async updateApp(appId: string, updateAppDto: any) { + const updatedApp = await this.prisma.client.app.update({ + where: {id: appId, deletedAt: { not : null }}, + data: {...updateAppDto}, + }); + + if (!updatedApp) { + return new HttpException( + `Could not update app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); } - async updateApp(appId: string, updateAppDto: any) { - const updatedApp = await this.prisma.client.app.update({ - where: { id: appId }, - data: updateAppDto, - }); - - if (!updatedApp) { - return new HttpException( - `Could not update app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } + return updatedApp; + } - return updatedApp; + async deleteApp(appId: string) { + const deletedAt = new Date() + + const deletedApp = await this.prisma.client.app.update({ + where: { id: appId, deletedAt: { not: null } }, + data: { deletedAt }, + }); + + if (!deletedApp) { + return new HttpException( + `Could not delete app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); } + return deletedApp; + } - async deleteApp(appId: string) { - const deletedApp = await this.prisma.client.app.update({ - where: { id: appId }, - data: { - deletedAt: new Date(), - }, - }); - - if (!deletedApp) { - return new HttpException( - `Could not delete app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } - return deletedApp; + async createAppRule( + appId: string, + ruleName: string, + createData: string[] + ) { + const { id: ruleId } = await this.getRuleType(ruleName); + + const data = createData.map((value: string) => ({ appId, ruleId, value })); + + const newAppRule = await this.prisma.client.appRule.createMany({ data }); + + if (!newAppRule) { + return new HttpException( + `Could not create app rule for this app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + return newAppRule; + } + + async updateAppRule( + appId: string, + ruleName: string, + updateData: string[], + ) { + const { id: ruleId } = await this.getRuleType(ruleName); + + const data = updateData.map((value) => ({ value })); + + const updatedAppRule = await this.prisma.client.appRule.updateMany({ + where: { ruleId, appId, deletedAt: { not: null } }, + data, + }); + + if (!updatedAppRule) { + return new HttpException( + `Could not update app rule for this app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); } - async createAppRule(appId: string, createAppRuleDto: any) { - const newAppRule = await this.prisma.client.appRule.create({ - data: { - appId, - ...createAppRuleDto, - }, - }); - if (!newAppRule) { - return new HttpException( - `Could not create app rule for this app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } - return newAppRule; + return updatedAppRule; + } + + async deleteAppRule(appId: string, ruleName: string) { + const { id: ruleId } = await this.getRuleType(ruleName); + const deletedAt = new Date(); + + const deletedAppRule = await this.prisma.client.appRule.updateMany({ + where: { ruleId, appId, deletedAt: { not: null } }, + data: { deletedAt }, + }); + + if (!deletedAppRule) { + return new HttpException( + `Could not delete app rule for this app`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); } + return deletedAppRule; + } - async updateAppRule(appId: string, ruleId: string, updateAppRuleDto: any) { - const updatedAppRule = await this.prisma.client.appRule.update({ - where: { id: ruleId, appId }, - data: updateAppRuleDto, - }); - - if (!updatedAppRule) { - return new HttpException( - `Could not update app rule for this app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } + async batchUpdateAppRules( + appId: string, + ruleName: string, + updateAppRuleDto: string[], + ) { - return updatedAppRule; + const { id: ruleId, validationType, validationValue } = await this.getRuleType(ruleName); + + + + if (!ruleId || ruleName === 'secret-key' || !ruleName) { + throw new HttpException( + 'Attempted to update invalid rule type', + HttpStatus.BAD_REQUEST, + ); } - async deleteAppRule(appId: string, ruleId: string) { - const deletedAppRule = await this.prisma.client.appRule.update({ - where: { id: ruleId, appId }, - data: { - deletedAt: new Date(), - }, - }); - - if (!deletedAppRule) { - return new HttpException( - `Could not delete app rule for this app`, - HttpStatus.INTERNAL_SERVER_ERROR, - ); + const existingAppRules = await this.prisma.client.appRule.findMany({ + where: { appId, ruleId, deletedAt: null }, + }); + + + + // Filter out new rules that are not in existingAppRules + const newAppRules = updateAppRuleDto?.filter( + (updateRule) => + !existingAppRules.some( + (existingRule: AppRule) => existingRule.value === updateRule, + ), + ); + + + + // Filter out existing rules that are not in updateData + const deleteAppRules = existingAppRules?.filter( + (existingRule: AppRule) => + !updateAppRuleDto.some( + (updateRule: string) => existingRule.value === updateRule, + ), + ); + + + + const ruleIdsToDelete = deleteAppRules.map((rule: any) => rule.id); + const ruleDataToCreate = newAppRules.map((newRule) => ({ + appId, + ruleId, + value: newRule, + })); + + + + if (validationType === 'regex') { + const regexExp = new RegExp(validationValue); + ruleDataToCreate.forEach((rule) => { + const matchResult = regexExp.test(rule.value); + if (!matchResult) { + throw new HttpException( + `Regex match failed for value: ${rule.value}`, + HttpStatus.BAD_REQUEST, + ); } - return deletedAppRule; + }); } - async batchUpdateAppRules( - appId: string, - updateAppRuleDto: { ruleId: string; data: string[] }[], - ) { - // only support one ruleId at this time - const { ruleId, data: updateData } = updateAppRuleDto[0]; - - const ruleType = await this.prisma.client.ruleType.findFirstOrThrow({ - where: { id: ruleId }, - }); - - if ( - !ruleType || - ruleType.id === 'secret-key' || - ruleType.name === 'secret-key' - ) { - throw new HttpException( - 'Attempted to update invalid rule type', - HttpStatus.BAD_REQUEST, - ); - } + await this.prisma.client.appRule.updateMany({ + where: { + appId, + ruleId, + id: { + in: ruleIdsToDelete, + }, + }, + data: { + deletedAt: new Date(), + }, + }); - if (ruleType.id !== ruleId) { - throw new HttpException( - 'Rule type does not match ruleId', - HttpStatus.BAD_REQUEST, - ); - } - const existingAppRules = await this.prisma.client.appRule.findMany({ - where: { appId, ruleId }, - }); - - // Filter out new rules that are not in existingAppRules - const newAppRules = updateData.filter( - (updateRule) => - !existingAppRules.some( - (existingRule: AppRule) => existingRule.value === updateRule, - ), - ); - - // Filter out existing rules that are not in updateData - const deleteAppRules = existingAppRules.filter( - (existingRule: AppRule) => - !updateData.some( - (updateRule: string) => existingRule.value === updateRule, - ), - ); - - const ruleIdsToDelete = deleteAppRules.map((rule: any) => rule.id); - - const ruleDataToCreate = newAppRules.map((newRule) => ({ - appId, - ruleId, - value: newRule, - })); - - if (ruleType.validationType === 'regex') { - const regexExp = new RegExp(ruleType.validationValue); - ruleDataToCreate.forEach((rule) => { - const matchResult = regexExp.test(rule.value); - if (!matchResult) { - throw new HttpException( - `Regex match failed for value: ${rule.value}`, - HttpStatus.BAD_REQUEST, - ); - } - }); - } - await this.prisma.client.appRule.updateMany({ - where: { - appId: appId, - ruleId: ruleId, - id: { - in: ruleIdsToDelete, - }, - }, - data: { - deletedAt: new Date(), - }, - }); - - const updatedAppRules = await this.prisma.client.appRule.createMany({ - data: ruleDataToCreate, - }); - - return updatedAppRules; + const updatedAppRules = await this.prisma.client.appRule.createMany({ + data: ruleDataToCreate, + }); + + return updatedAppRules; + } + + + + async updateSecretKeyRule(appId: string, action: 'generate' | 'delete') { + const { id: ruleId } = await this.getRuleType('secret-key'); + + + const secretIdExists = await this.prisma.client.appRule.findFirst({ + where: { appId, ruleId }, + }); + + if (action === 'delete' && secretIdExists) { + const deleteSecretKey = await this.prisma.client.appRule.delete({ + where: { id: secretIdExists.id }, + }); + + if (deleteSecretKey) { + return { delete: true }; + } } - async updateSecretKeyRule(appId: string, action: 'generate' | 'delete') { - const ruleType = await this.prisma.client.ruleType.findFirstOrThrow({ - where: { id: 'secret-key' }, - }); + if (secretIdExists && action === 'generate') { - const secretIdExists = await this.prisma.client.appRule.findFirst({ - where: { appId, ruleId: ruleType.id }, - }); + const { secretKey, hashedKey } = this.generateSecretKey() - if (action === 'delete' && secretIdExists) { - const deleteSecretKey = await this.prisma.client.appRule.delete({ - where: { id: secretIdExists.id }, - }); + const updateSecretKey = await this.prisma.client.appRule.update({ + where: { id: secretIdExists.id }, + data: { + value: hashedKey, + }, + }); - if (deleteSecretKey) { - return { delete: true }; - } - } + if (updateSecretKey) { + return { key: secretKey }; + } + } else if (!secretIdExists && action === 'generate') { - if (secretIdExists && action === 'generate') { - const secretKey = randomBytes(8).toString('hex'); - const hashedKey = createHash('sha256').update(secretKey).digest('hex'); - - const updateSecretKey = await this.prisma.client.appRule.update({ - where: { id: secretIdExists.id }, - data: { - value: hashedKey, - }, - }); - - if (updateSecretKey) { - return { key: secretKey }; - } - } else if (!secretIdExists && action === 'generate') { - const secretKey = randomBytes(8).toString('hex'); - const hashedKey = createHash('sha256').update(secretKey).digest('hex'); - - const newSecretKey = await this.prisma.client.appRule.create({ - data: { - appId, - ruleId: ruleType.id, - value: hashedKey, - }, - }); - - if (newSecretKey) { - return { key: secretKey }; - } - } + const { secretKey, hashedKey } = this.generateSecretKey() + + const newSecretKey = await this.prisma.client.appRule.create({ + data: { + appId, + ruleId, + value: hashedKey, + }, + }); + + if (newSecretKey) { + return { key: secretKey }; + } + } + } + + private generateSecretKey() { + const secretKey = randomBytes(8).toString('hex'); + const hashedKey = createHash('sha256').update(secretKey).digest('hex'); + + return { secretKey, hashedKey } + } + + + + async verifyAppAccess(req: Request, appId: string) { + const userAddress = await this.userService.getUserAddress(req) + const apps = await this.getAppsByUser(userAddress) + + const appIds = new Set(apps.map((app: any) => app.id)) + const hasAccess = appIds.has(appId) + + if (!hasAccess) { + throw new HttpException(`User does not have access to this app`, HttpStatus.UNAUTHORIZED) } + + return hasAccess; + + } } diff --git a/web-portal/backend/src/decorator/public.decorator.ts b/web-portal/backend/src/decorator/public.decorator.ts new file mode 100644 index 00000000..b3845e12 --- /dev/null +++ b/web-portal/backend/src/decorator/public.decorator.ts @@ -0,0 +1,4 @@ +import { SetMetadata } from '@nestjs/common'; + +export const IS_PUBLIC_KEY = 'isPublic'; +export const Public = () => SetMetadata(IS_PUBLIC_KEY, true); diff --git a/web-portal/backend/src/guards/auth.guard.ts b/web-portal/backend/src/guards/auth.guard.ts index 5223cbd8..e79a6ee5 100644 --- a/web-portal/backend/src/guards/auth.guard.ts +++ b/web-portal/backend/src/guards/auth.guard.ts @@ -1,13 +1,36 @@ import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common'; +import { Reflector } from '@nestjs/core' import { unsealData } from 'iron-session'; -import { ISession, SESSION_OPTIONS } from '../siwe/siwe.service'; +import { ISession, SESSION_OPTIONS, SiweService } from '../siwe/siwe.service'; +import { IS_PUBLIC_KEY } from '../decorator/public.decorator' +import { AppsService } from '../apps/apps.service'; @Injectable() export class AuthGuard implements CanActivate { + constructor( + private readonly siweService: SiweService, + private readonly appsService: AppsService, + private readonly reflector: Reflector + ) { } async canActivate(context: ExecutionContext) { + + const isPublic = this.reflector.getAllAndOverride(IS_PUBLIC_KEY, [ + context.getHandler(), + context.getClass() + ]) + + + if (isPublic) { + return true; + } + const request = context.switchToHttp().getRequest(); const sessionCookie = request.cookies?.session; + // read params + const tenantIdFromParams = request.params['tenantId'] + const appIdFromParams = request.params['appId'] + if (!sessionCookie) { return false; } @@ -17,8 +40,26 @@ export class AuthGuard implements CanActivate { SESSION_OPTIONS, ); + // no address or chainId if (!address || !chainId) return false; + + const session = await this.siweService.getSession(sessionCookie) + + // fail if no session + if (!session) return false + + // verify appId access + if (appIdFromParams) { + const hasAccessToAppId = await this.appsService.verifyAppAccess(request, appIdFromParams); + if (!hasAccessToAppId) return false + } + + // verify tenantId access + if (tenantIdFromParams) { + if (tenantIdFromParams !== session?.tenantId) { return false } + } + return true; } } diff --git a/web-portal/backend/src/main.ts b/web-portal/backend/src/main.ts index b3833ca1..0f6ea2a2 100644 --- a/web-portal/backend/src/main.ts +++ b/web-portal/backend/src/main.ts @@ -1,4 +1,5 @@ import { HttpAdapterHost, NestFactory } from '@nestjs/core'; +import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'; import { PrismaClientExceptionFilter } from 'nestjs-prisma'; import { AppModule } from './app.module'; import * as cookieParser from 'cookie-parser'; @@ -8,12 +9,17 @@ async function bootstrap() { // const { httpAdapter } = app.get(HttpAdapterHost); // app.useGlobalFilters(new PrismaClientExceptionFilter(httpAdapter)); - // Remove CORS in prod - - app.enableCors(); + const config = new DocumentBuilder() + .setTitle('Gateway Web-Portal Backend') + .setDescription('Backend APIs') + .setVersion('1.0') + .addTag('specs') + .build(); + const document = SwaggerModule.createDocument(app, config); + SwaggerModule.setup('specs', app, document); app.use(cookieParser()); - await app.listen(process.env.PORT || 4000, '0.0.0.0'); + await app.listen(process.env.PORT || 4000); } bootstrap(); diff --git a/web-portal/backend/src/org/org.controller.ts b/web-portal/backend/src/org/org.controller.ts index a9d70a03..04e036e5 100644 --- a/web-portal/backend/src/org/org.controller.ts +++ b/web-portal/backend/src/org/org.controller.ts @@ -1,6 +1,4 @@ -import { Controller, UseGuards } from '@nestjs/common'; -import { AuthGuard } from '../guards/auth.guard'; +import { Controller } from '@nestjs/common'; @Controller('org') -@UseGuards(AuthGuard) -export class OrgController {} +export class OrgController { } diff --git a/web-portal/backend/src/siwe/siwe.controller.ts b/web-portal/backend/src/siwe/siwe.controller.ts index 51f7721a..ce741a8c 100644 --- a/web-portal/backend/src/siwe/siwe.controller.ts +++ b/web-portal/backend/src/siwe/siwe.controller.ts @@ -7,14 +7,15 @@ import { Res, HttpStatus, Delete, - UseGuards, } from '@nestjs/common'; import { SiweService } from './siwe.service'; import { Request, Response } from 'express'; -import { AuthGuard } from '../guards/auth.guard'; +import { Public } from '../decorator/public.decorator'; + @Controller('siwe') +@Public() export class SiweController { - constructor(private readonly siweService: SiweService) {} + constructor(private readonly siweService: SiweService) { } @Get() async getSession( @@ -65,7 +66,6 @@ export class SiweController { } @Delete() - @UseGuards(AuthGuard) async signOut(@Req() request: Request, @Res() response: Response) { const sessionCookie = request?.cookies['session']; if (sessionCookie) { diff --git a/web-portal/backend/src/siwe/siwe.service.ts b/web-portal/backend/src/siwe/siwe.service.ts index f6f8400f..0d9bb55f 100644 --- a/web-portal/backend/src/siwe/siwe.service.ts +++ b/web-portal/backend/src/siwe/siwe.service.ts @@ -20,7 +20,7 @@ export const SESSION_OPTIONS = { @Injectable() export class SiweService { - constructor(private readonly userService: UserService) {} + constructor(private readonly userService: UserService) { } async getSession(sessionCookie: string) { const { address, chainId } = await unsealData( diff --git a/web-portal/backend/src/tenant/tenant.controller.ts b/web-portal/backend/src/tenant/tenant.controller.ts index 571327c7..6723861f 100644 --- a/web-portal/backend/src/tenant/tenant.controller.ts +++ b/web-portal/backend/src/tenant/tenant.controller.ts @@ -1,12 +1,10 @@ -import { Controller, Get, Post, Param, Query, UseGuards } from '@nestjs/common'; +import { Controller, Get, Post, Param, Query } from '@nestjs/common'; import { TenantService } from './tenant.service'; -import { AuthGuard } from '../guards/auth.guard'; @Controller('tenant') -@UseGuards(AuthGuard) export class TenantController { - constructor(private readonly tenantService: TenantService) {} + constructor(private readonly tenantService: TenantService) { } @Post() async createTenant() { @@ -21,15 +19,15 @@ export class TenantController { return validation; } - @Get(':id') - async getTenantById(@Param('id') id: string) { + @Get(':tenantId') + async getTenantById(@Param('tenantId') tenantId: string) { // @note: This action returns a tenant by its id - return this.tenantService.getTenantById(id); + return this.tenantService.getTenantById(tenantId); } - @Get(':id/billing') - async getTenantBillingHistory(@Param('id') id: string) { + @Get(':tenantId/billing') + async getTenantBillingHistory(@Param('tenantId') tenantId: string) { // @note: This action returns billing history for a tenant - return this.tenantService.getTenantBillingHistory(id); + return this.tenantService.getTenantBillingHistory(tenantId); } } diff --git a/web-portal/backend/src/usage/usage.controller.spec.ts b/web-portal/backend/src/usage/usage.controller.spec.ts new file mode 100644 index 00000000..b9e95488 --- /dev/null +++ b/web-portal/backend/src/usage/usage.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { UsageController } from './usage.controller'; + +describe('UsageController', () => { + let controller: UsageController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [UsageController], + }).compile(); + + controller = module.get(UsageController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/web-portal/backend/src/usage/usage.controller.ts b/web-portal/backend/src/usage/usage.controller.ts new file mode 100644 index 00000000..3d3acc64 --- /dev/null +++ b/web-portal/backend/src/usage/usage.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Param } from '@nestjs/common'; +import { UsageService } from './usage.service'; + + + +@Controller('usage') +export class UsageController { + constructor(private readonly usageService: UsageService) { } + @Get('/app/:appId/:period') + async getAppUsage( + @Param('appId') appId: string, + @Param('period') period: string, + ) { + return this.usageService.getAppUsage(appId, period); + } + + @Get('/tenant/:tenantId/:period') + async getTenantUsage( + @Param('tenantId') tenantId: string, + @Param('period') period: string, + ) { + return this.usageService.getTenantUsage(tenantId, period); + } +} diff --git a/web-portal/backend/src/usage/usage.module.ts b/web-portal/backend/src/usage/usage.module.ts new file mode 100644 index 00000000..49e530a3 --- /dev/null +++ b/web-portal/backend/src/usage/usage.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { UsageService } from './usage.service'; +import { UsageController } from './usage.controller'; + +@Module({ + providers: [UsageService], + controllers: [UsageController] +}) +export class UsageModule {} diff --git a/web-portal/backend/src/usage/usage.service.spec.ts b/web-portal/backend/src/usage/usage.service.spec.ts new file mode 100644 index 00000000..f25ab6ef --- /dev/null +++ b/web-portal/backend/src/usage/usage.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { UsageService } from './usage.service'; + +describe('UsageService', () => { + let service: UsageService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [UsageService], + }).compile(); + + service = module.get(UsageService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/web-portal/backend/src/usage/usage.service.ts b/web-portal/backend/src/usage/usage.service.ts new file mode 100644 index 00000000..9b35f588 --- /dev/null +++ b/web-portal/backend/src/usage/usage.service.ts @@ -0,0 +1,72 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { createHash } from 'crypto'; + +@Injectable() +export class UsageService { + async getAppUsage(appId: string, period: string): Promise { + const hashedAppId = createHash('sha256').update(appId).digest('hex'); + + const start = period + const step = this.getStep(period) + const q = `gateway_relay_usage{appId="${hashedAppId}"}`; + + if (!step || !period) { + throw new HttpException(`Couldn't get tenant data`, HttpStatus.BAD_REQUEST) + } + + const result = await this.fetchUsageData(q, start, step); + return result.json(); + } + + async getTenantUsage(tenantId: string, period: string): Promise { + + const q = `gateway_relay_usage{tenant="${tenantId}"}`; + const start = period + const step = this.getStep(period) + + if (!step || !period) { + throw new HttpException(`Couldn't get tenant data`, HttpStatus.BAD_REQUEST) + } + + const result = await this.fetchUsageData(q, start, step); + return result.json(); + } + + private async fetchUsageData( + query: string, + start: string, + step: number | string, + ): Promise { + + const url = process.env.PROM_URL +`query_range?query=sum(increase(${query}))&start=${start}&end=now&step=${step}`; + // do include `/` into url + const result = await fetch(url, { + headers: { + Authorization: String(process.env.PROM_TOKEN), + }, + }); + + if (!result.ok) { + throw new HttpException('Failed to fetch data', result.status); + } + + return result; + } + + + private getStep(period: string): string | null { + switch (period) { + case '24h': + return '1h'; + case '1h': + return '5m'; + case '7d': + return '1d'; + case '30d': + return '1d'; + default: + return null; + } + } + +} diff --git a/web-portal/backend/src/user/user.service.ts b/web-portal/backend/src/user/user.service.ts index 431c61cf..27afc501 100644 --- a/web-portal/backend/src/user/user.service.ts +++ b/web-portal/backend/src/user/user.service.ts @@ -3,111 +3,154 @@ import { CustomPrismaService } from 'nestjs-prisma'; import { PrismaClient } from '@/.generated/client'; import { createHash } from 'crypto'; import { TenantService } from '../tenant/tenant.service'; +import { unsealData } from 'iron-session'; +import { Request } from 'express'; +import { ISession, SESSION_OPTIONS } from '../siwe/siwe.service'; + @Injectable() export class UserService { - constructor( - @Inject('Postgres') - private prisma: CustomPrismaService, // <-- Inject the PrismaClient - private tenantService: TenantService, - ) {} - - async getOrCreate(ethAddress: string) { - const existingUser = await this.prisma.client.user.findUnique({ - where: { - ethAddress: createHash('sha256').update(ethAddress).digest('hex'), - }, - include: { - orgs: {}, - }, - }); - - if (!existingUser || existingUser?.orgs?.length === 0) { - // @note: this will generate enterprise + tenant before creating a new user; - const { enterpriseId } = await this.tenantService.create(); // <- show this secret for the first time user to backup - - if (!enterpriseId) { - throw new HttpException( - 'Unable to create tenant', - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } - - if (existingUser) { - const updateExistingUser = await this.prisma.client.user.update({ - where: { - id: existingUser.id, - }, - data: { - orgs: { - create: { - enterprise: { - connect: { - id: enterpriseId, - }, - }, - }, - }, - }, + constructor( + @Inject('Postgres') + private prisma: CustomPrismaService, // <-- Inject the PrismaClient + private tenantService: TenantService, + ) { } - include: { - orgs: {}, - }, + async getOrCreate(ethAddress: string) { + const existingUser = await this.prisma.client.user.findUnique({ + where: { + ethAddress: createHash('sha256').update(ethAddress).digest('hex'), + }, + include: { + orgs: {}, + }, }); - const { id, active, createdAt, orgs } = updateExistingUser; - - const tenantId = await this.getTenantIdByEnterpriseId(enterpriseId); - return { id, active, createdAt, orgs, tenantId }; - } - const newUser = await this.prisma.client.user.create({ - data: { - ethAddress: createHash('sha256').update(ethAddress).digest('hex'), - orgs: { - create: { - enterprise: { - connect: { - id: enterpriseId, + + if (!existingUser || existingUser?.orgs?.length === 0) { + // @note: this will generate enterprise + tenant before creating a new user; + const { enterpriseId } = await this.tenantService.create(); // <- show this secret for the first time user to backup + + if (!enterpriseId) { + throw new HttpException( + 'Unable to create tenant', + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + + if (existingUser) { + const updateExistingUser = await this.prisma.client.user.update({ + where: { + id: existingUser.id, + }, + data: { + orgs: { + create: { + enterprise: { + connect: { + id: enterpriseId, + }, + }, + }, + }, + }, + + include: { + orgs: {}, + }, + }); + const { id, active, createdAt, orgs } = updateExistingUser; + + const tenantId = await this.getTenantIdByEnterpriseId(enterpriseId); + return { id, active, createdAt, orgs, tenantId }; + } + const newUser = await this.prisma.client.user.create({ + data: { + ethAddress: createHash('sha256').update(ethAddress).digest('hex'), + orgs: { + create: { + enterprise: { + connect: { + id: enterpriseId, + }, + }, + }, + }, + }, + include: { + orgs: {}, }, - }, + }); + + if (!newUser) + throw new HttpException( + 'Unable to create tenant', + HttpStatus.INTERNAL_SERVER_ERROR, + ); + + const { id, active, createdAt, orgs } = newUser; + + const tenantId = await this.getTenantIdByEnterpriseId(enterpriseId); + + return { id, active, createdAt, orgs, tenantId, netBalance: 0 }; + } + + const { id, active, createdAt, orgs } = existingUser; + + const tenantId = await this.getTenantIdByEnterpriseId(orgs[0].enterpriseId); + + const netBalance = await this.getTenantBalance(tenantId); + + return { id, active, createdAt, orgs, tenantId, netBalance }; + } + + async getTenantIdByEnterpriseId(enterpriseId: string) { + const tenants = await this.prisma.client.tenant.findMany({ + where: { + enterpriseId, }, - }, - }, - include: { - orgs: {}, - }, - }); - - if (!newUser) - throw new HttpException( - 'Unable to create tenant', - HttpStatus.INTERNAL_SERVER_ERROR, - ); + }); - const { id, active, createdAt, orgs } = newUser; + if (!tenants) + throw new HttpException( + 'Unable to find tenant', + HttpStatus.INTERNAL_SERVER_ERROR, + ); + + return tenants[0].id; + } - const tenantId = await this.getTenantIdByEnterpriseId(enterpriseId); + async getTenantBalance(tenantId: string) { + const netBalance = await this.prisma.client.$queryRaw` + SELECT payment.balance - relay.usage as net FROM + (SELECT + COALESCE(SUM(case when "transactionType"='CREDIT' then amount else 0 end) - + SUM(case when "transactionType"='DEBIT' then amount else 0 end), 0) + AS balance FROM "PaymentLedger" WHERE "tenantId" = ${tenantId}) as payment, + (SELECT + COALESCE(SUM(case when "transactionType"='CREDIT' then amount else 0 end) - + SUM(case when "transactionType"='DEBIT' then amount else 0 end), 0) + AS usage FROM "RelayLedger" WHERE "tenantId" = ${tenantId}) as relay + `; - return { id, active, createdAt, orgs, tenantId }; + return netBalance; } - const { id, active, createdAt, orgs } = existingUser; + async getUserAddress(req: Request) { - const tenantId = await this.getTenantIdByEnterpriseId(orgs[0].enterpriseId); - return { id, active, createdAt, orgs, tenantId }; - } + const sessionCookie = req.cookies?.session; - async getTenantIdByEnterpriseId(enterpriseId: string) { - const tenants = await this.prisma.client.tenant.findMany({ - where: { - enterpriseId, - }, - }); + if (!sessionCookie) { + throw new HttpException(`Unauthorized Attempt`, HttpStatus.UNAUTHORIZED) + } - if (!tenants) - throw new HttpException( - 'Unable to find tenant', - HttpStatus.INTERNAL_SERVER_ERROR, - ); + const { address } = await unsealData( + sessionCookie, + SESSION_OPTIONS, + ); + + if (!address) { + throw new HttpException(`Couldn't decode user address`, HttpStatus.BAD_REQUEST) + } - return tenants[0].id; - } + return address + } } diff --git a/web-portal/backend/src/utils/utils.controller.ts b/web-portal/backend/src/utils/utils.controller.ts index 75ae4c17..24d9ba26 100644 --- a/web-portal/backend/src/utils/utils.controller.ts +++ b/web-portal/backend/src/utils/utils.controller.ts @@ -1,9 +1,7 @@ -import { Controller, Get, UseGuards, Param, Body } from '@nestjs/common'; -import { AuthGuard } from '../guards/auth.guard'; +import { Controller, Get, Param, Body } from '@nestjs/common'; import { UtilsService } from './utils.service'; @Controller('utils') -@UseGuards(AuthGuard) export class UtilsController { constructor(private readonly utilsService: UtilsService) { } diff --git a/web-portal/backend/src/utils/utils.service.ts b/web-portal/backend/src/utils/utils.service.ts index 8928f90e..f5633591 100644 --- a/web-portal/backend/src/utils/utils.service.ts +++ b/web-portal/backend/src/utils/utils.service.ts @@ -3,7 +3,7 @@ import { CustomPrismaService } from 'nestjs-prisma'; import { PrismaClient, TransactionType } from '@/.generated/client'; import { Cron, CronExpression } from '@nestjs/schedule'; import { parseAbiItem, fromHex, isAddress } from 'viem'; -import { viemClient } from './viemClient'; +import { opClient, baseClient, gnosisClient } from './viemClient'; interface IParsedLog { tenantId: string; @@ -22,7 +22,7 @@ export class UtilsService { constructor( @Inject('Postgres') private prisma: CustomPrismaService, // <-- Inject the PrismaClient - ) {} + ) { } async getChains() { const chains = this.prisma.client.products.findMany({ @@ -168,39 +168,64 @@ export class UtilsService { @Cron(CronExpression.EVERY_MINUTE) async watchEvent() { - console.log('Getting Event Logs'); - const blockNumber = await viemClient.getBlockNumber(); - const logs = await viemClient.getLogs({ - event, - address: portrAddress, - fromBlock: blockNumber - BigInt(1000), - toBlock: blockNumber, - }); + console.log('Getting Event Logs from all clients'); + + // Define clients and their respective names + const clients = [ + { client: opClient, name: 'optimism' }, + { client: gnosisClient, name: 'gnosis' }, + { client: baseClient, name: 'base' } + ]; + + // Fetch and parse logs for all clients + const allParsedLogs = await Promise.all( + clients.map(async ({ client, name }) => { + console.log(`Getting Event Logs from ${name}`); + const blockNumber = await client.getBlockNumber(); + const logs = await client.getLogs({ + event, + address: portrAddress, + fromBlock: blockNumber - BigInt(1000), + toBlock: blockNumber, + }); + + return this.parseLogs(logs, name); + }) + ); - const parsedLogs: IParsedLog[] = logs.map((log: any) => ({ - tenantId: fromHex(log?.args?._identifier, 'string').replaceAll( - `\x00`, - '', - ), - amount: Number(log?.args?._amount), - referenceId: log.transactionHash!, - transactionType: TransactionType.CREDIT!, - })); + const [parsedLogsOP, parsedLogsGnosis, parsedLogsBase] = allParsedLogs; - console.log({ parsedLogs }); + console.log({ parsedLogsOP, parsedLogsGnosis, parsedLogsBase }); - if (!parsedLogs) console.log('No New Redemptions'); + if (!parsedLogsOP.length && !parsedLogsGnosis.length && !parsedLogsBase.length) { + console.log('No New Redemptions'); + return; + } // Create records for unique logs const appliedLogs = await this.prisma.client.paymentLedger.createMany({ skipDuplicates: true, - data: parsedLogs, + data: [...parsedLogsOP, ...parsedLogsGnosis, ...parsedLogsBase], }); console.log({ appliedLogs }); - if (!appliedLogs) console.log('Error Applying logs'); + if (!appliedLogs) { + console.log('Error Applying logs'); + } else { + console.log('Applied New logs'); + } + } - console.log('Applied New logs'); + // Helper function to parse logs + parseLogs(logs: any[], network: string): IParsedLog[] { + return logs.map((log: any) => ({ + tenantId: fromHex(log?.args?._identifier, 'string').replaceAll(`\x00`, ''), + amount: Number(log?.args?._amount * 10 ** -12), + // 10 ** -18 (to parse to human readable) * 10 ** 3 (for 1000 relay per token) * 10 ** 3 for chain weight = 10 ** -12 + referenceId: network + `:` + log.transactionHash!, + transactionType: TransactionType.CREDIT!, + })); } + } diff --git a/web-portal/backend/src/utils/viemClient.ts b/web-portal/backend/src/utils/viemClient.ts index f0db7b8f..2065add3 100644 --- a/web-portal/backend/src/utils/viemClient.ts +++ b/web-portal/backend/src/utils/viemClient.ts @@ -1,7 +1,17 @@ import { createPublicClient, http } from 'viem'; -import { optimism } from 'viem/chains'; +import { optimism, base, gnosis } from 'viem/chains'; -export const viemClient = createPublicClient({ +export const opClient = createPublicClient({ chain: optimism, transport: http(), }); + +export const baseClient = createPublicClient({ + chain: base, + transport: http(), +}); + +export const gnosisClient = createPublicClient({ + chain: gnosis, + transport: http(), +}); diff --git a/web-portal/frontend/components/apps/appRuleModal.tsx b/web-portal/frontend/components/apps/appRuleModal.tsx index b3f81f5c..47322189 100644 --- a/web-portal/frontend/components/apps/appRuleModal.tsx +++ b/web-portal/frontend/components/apps/appRuleModal.tsx @@ -30,7 +30,7 @@ export default function CreateAppRuleModal() { {shouldOpen === "secret-key" && } {shouldOpen === "approved-chains" && } {shouldOpen === "allowed-user-agents" && } - {shouldOpen === "rate-limit" && } + {shouldOpen === "rate-limits" && } ); } diff --git a/web-portal/frontend/components/apps/appRules.tsx b/web-portal/frontend/components/apps/appRules.tsx index 13fe1273..cf9c9b66 100644 --- a/web-portal/frontend/components/apps/appRules.tsx +++ b/web-portal/frontend/components/apps/appRules.tsx @@ -23,12 +23,7 @@ const AppRules: React.FC<{ rule: Partial }> = ({ rule }) => { const values = existingData?.map((item) => ( {item.value ?.replace("P1D", "Daily") diff --git a/web-portal/frontend/components/apps/apptabs.tsx b/web-portal/frontend/components/apps/apptabs.tsx index aa189b33..732c3b7f 100644 --- a/web-portal/frontend/components/apps/apptabs.tsx +++ b/web-portal/frontend/components/apps/apptabs.tsx @@ -25,7 +25,7 @@ const AppTabs: React.FC = () => { Insights Endpoints - Usage + {/* TOOD: Future- add logs */} Rules @@ -35,9 +35,6 @@ const AppTabs: React.FC = () => { - - - {ruleTypes.map((rule: IRuleType) => ( diff --git a/web-portal/frontend/components/apps/forms/allowedOriginsForm.tsx b/web-portal/frontend/components/apps/forms/allowedOriginsForm.tsx index 04b6fddc..49cbfb85 100644 --- a/web-portal/frontend/components/apps/forms/allowedOriginsForm.tsx +++ b/web-portal/frontend/components/apps/forms/allowedOriginsForm.tsx @@ -46,12 +46,7 @@ export default function AllowedOriginsForm() { key={item} withRemoveButton onRemove={() => handleValueRemove(item)} - size="lg" m={2} - bg={"blue"} - style={{ - color: "white", - }} > {item} @@ -65,11 +60,9 @@ export default function AllowedOriginsForm() { placeholder="Enter a valid Url" type="url" inputWrapperOrder={["label", "input", "description"]} - style={{ width: "100%" }} {...form.getInputProps("url")} />