diff --git a/.github/workflows/container-hepa-aws.yaml b/.github/workflows/container-hepa-aws.yaml new file mode 100644 index 000000000..336af4321 --- /dev/null +++ b/.github/workflows/container-hepa-aws.yaml @@ -0,0 +1,52 @@ +name: container-hepa-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: hepa + +jobs: + container-hepa-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/hepa/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-hepa-ghcr.yaml b/.github/workflows/container-hepa-ghcr.yaml new file mode 100644 index 000000000..bcb3269d9 --- /dev/null +++ b/.github/workflows/container-hepa-ghcr.yaml @@ -0,0 +1,54 @@ +name: container-hepa-ghcr +on: + push: + branches: + - main + - bnewbold/automod +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-hepa-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=hepa:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/hepa/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/HACKING.md b/HACKING.md index f36821aa9..ef36c48c9 100644 --- a/HACKING.md +++ b/HACKING.md @@ -13,6 +13,7 @@ Run with, eg, `go run ./cmd/bigsky`): - `cmd/fakermaker`: helper to generate fake accounts and content for testing - `cmd/supercollider`: event stream load generation tool - `cmd/sonar`: event stream monitoring tool +- `cmd/hepa`: auto-moderation rule engine service - `gen`: dev tool to run CBOR type codegen Packages: @@ -23,6 +24,7 @@ Packages: - `atproto/crypto`: crytographic helpers (signing, key generation and serialization) - `atproto/syntax`: string types and parsers for identifiers, datetimes, etc - `atproto/identity`: DID and handle resolution +- `automod`: moderation and anti-spam rules engine - `bgs`: server implementation for crawling, etc - `carstore`: library for storing repo data in CAR files on disk, plus a metadata SQL db - `events`: types, codegen CBOR helpers, and persistence for event feeds diff --git a/Makefile b/Makefile index d0b3b44cb..d9a85b8da 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ build: ## Build all executables go build ./cmd/stress go build ./cmd/fakermaker go build ./cmd/labelmaker + go build ./cmd/hepa go build ./cmd/supercollider go build -o ./sonar-cli ./cmd/sonar go build ./cmd/palomar diff --git a/api/atproto/servercreateSession.go b/api/atproto/servercreateSession.go index ac197250c..77beaf087 100644 --- a/api/atproto/servercreateSession.go +++ b/api/atproto/servercreateSession.go @@ -7,7 +7,6 @@ package atproto import ( "context" - "github.com/bluesky-social/indigo/lex/util" "github.com/bluesky-social/indigo/xrpc" ) @@ -20,13 +19,13 @@ type ServerCreateSession_Input struct { // ServerCreateSession_Output is the output of a com.atproto.server.createSession call. type ServerCreateSession_Output struct { - AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` - Did string `json:"did" cborgen:"did"` - DidDoc *util.LexiconTypeDecoder `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Did string `json:"did" cborgen:"did"` + //DidDoc *util.LexiconTypeDecoder `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` } // ServerCreateSession calls the XRPC method "com.atproto.server.createSession". diff --git a/atproto/syntax/aturi.go b/atproto/syntax/aturi.go index cdf340fc4..cd499fecf 100644 --- a/atproto/syntax/aturi.go +++ b/atproto/syntax/aturi.go @@ -62,14 +62,14 @@ func (n ATURI) Authority() AtIdentifier { // Returns path segment, without leading slash, as would be used in an atproto repository key. Or empty string if there is no path. func (n ATURI) Path() string { parts := strings.SplitN(string(n), "/", 5) - if len(parts) < 3 { + if len(parts) < 4 { // something has gone wrong (would not validate) return "" } - if len(parts) == 3 { - return parts[2] + if len(parts) == 4 { + return parts[3] } - return parts[2] + "/" + parts[3] + return parts[3] + "/" + parts[4] } // Returns a valid NSID if there is one in the appropriate part of the path, otherwise empty. diff --git a/atproto/syntax/aturi_test.go b/atproto/syntax/aturi_test.go index 2851e35ae..4ba278300 100644 --- a/atproto/syntax/aturi_test.go +++ b/atproto/syntax/aturi_test.go @@ -20,11 +20,20 @@ func TestInteropATURIsValid(t *testing.T) { if len(line) == 0 || line[0] == '#' { continue } - _, err := ParseATURI(line) + aturi, err := ParseATURI(line) if err != nil { fmt.Println("FAILED, GOOD: " + line) } assert.NoError(err) + + // check that Path() is working + col := aturi.Collection() + rkey := aturi.RecordKey() + if rkey != "" { + assert.Equal(col.String()+"/"+rkey.String(), aturi.Path()) + } else if col != "" { + assert.Equal(col.String(), aturi.Path()) + } } assert.NoError(scanner.Err()) } @@ -67,7 +76,22 @@ func TestATURIParts(t *testing.T) { rkey := uri.RecordKey() assert.Equal(parts[3], rkey.String()) } +} + +func TestATURIPath(t *testing.T) { + assert := assert.New(t) + uri1, err := ParseATURI("at://did:abc:123/io.nsid.someFunc/record-key") + assert.NoError(err) + assert.Equal("io.nsid.someFunc/record-key", uri1.Path()) + + uri2, err := ParseATURI("at://did:abc:123/io.nsid.someFunc") + assert.NoError(err) + assert.Equal("io.nsid.someFunc", uri2.Path()) + + uri3, err := ParseATURI("at://did:abc:123") + assert.NoError(err) + assert.Equal("", uri3.Path()) } func TestATURINormalize(t *testing.T) { @@ -93,5 +117,6 @@ func TestATURINoPanic(t *testing.T) { _ = bad.RecordKey() _ = bad.Normalize() _ = bad.String() + _ = bad.Path() } } diff --git a/automod/README.md b/automod/README.md new file mode 100644 index 000000000..3177f4318 --- /dev/null +++ b/automod/README.md @@ -0,0 +1,23 @@ +indigo/automod +============== + +This package (`github.com/bluesky-social/indigo/automod`) contains a "rules engine" to augment human moderators in the atproto network. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". A lot of what this package does is collect and maintain caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information. + +A primary design goal is to have a flexible framework to allow new rules to be written and deployed rapidly in response to new patterns of spam and abuse. + +Some example rules are included in the `automod/rules` package, but the expectation is that some real-world rules will be kept secret. + +Code for subscribing to a firehose is not included here; see `cmd/hepa` for a complete service built on this library. + + +## Design + +Prior art and inspiration: + +* The [SQRL language](https://sqrl-lang.github.io/sqrl/) and runtime was originally developed by an industry vendor named Smyte, then acquired by Twitter, with some core Javascript components released open source in 2023. The SQRL documentation is extensive and describes many of the design trade-offs and features specific to rules engines. Bluesky considered adopting SQRL but decided to start with a simpler runtime with rules in a known language (golang). + +* Reddit's [automod system](https://www.reddit.com/wiki/automoderator/) is simple an accessible for non-technical sub-reddit community moderators. Discord has a large ecosystem of bots which can help communities manage some moderation tasks, in particular mitigating spam and brigading. + +* Facebook's FXL and Haxl rule languages have been in use for over a decade. The 2012 paper ["The Facebook Immune System"](https://css.csail.mit.edu/6.858/2012/readings/facebook-immune.pdf) gives a good overview of design goals and how a rules engine fits in to a an overall anti-spam/anti-abuse pipeline. + +* Email anti-spam systems like SpamAssassin and rspamd have been modular and configurable for several decades. diff --git a/automod/account_meta.go b/automod/account_meta.go new file mode 100644 index 000000000..625205c04 --- /dev/null +++ b/automod/account_meta.go @@ -0,0 +1,126 @@ +package automod + +import ( + "context" + "encoding/json" + "fmt" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type ProfileSummary struct { + HasAvatar bool + Description *string + DisplayName *string +} + +type AccountPrivate struct { + Email string + EmailConfirmed bool + IndexedAt time.Time +} + +// information about a repo/account/identity, always pre-populated and relevant to many rules +type AccountMeta struct { + Identity *identity.Identity + Profile ProfileSummary + Private *AccountPrivate + AccountLabels []string + FollowersCount int64 + FollowsCount int64 + PostsCount int64 +} + +func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) { + + // wipe parsed public key; it's a waste of space and can't serialize + ident.ParsedPublicKey = nil + + // fallback in case client wasn't configured (eg, testing) + if e.BskyClient == nil { + e.Logger.Warn("skipping account meta hydration") + am := AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } + return &am, nil + } + + existing, err := e.Cache.Get(ctx, "acct", ident.DID.String()) + if err != nil { + return nil, err + } + if existing != "" { + var am AccountMeta + err := json.Unmarshal([]byte(existing), &am) + if err != nil { + return nil, fmt.Errorf("parsing AccountMeta from cache: %v", err) + } + am.Identity = ident + return &am, nil + } + + // fetch account metadata + pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String()) + if err != nil { + return nil, err + } + + var labels []string + for _, lbl := range pv.Labels { + labels = append(labels, lbl.Val) + } + + am := AccountMeta{ + Identity: ident, + Profile: ProfileSummary{ + HasAvatar: pv.Avatar != nil, + Description: pv.Description, + DisplayName: pv.DisplayName, + }, + AccountLabels: dedupeStrings(labels), + } + if pv.PostsCount != nil { + am.PostsCount = *pv.PostsCount + } + if pv.FollowersCount != nil { + am.FollowersCount = *pv.FollowersCount + } + if pv.FollowsCount != nil { + am.FollowsCount = *pv.FollowsCount + } + + if e.AdminClient != nil { + pv, err := comatproto.AdminGetAccountInfo(ctx, e.AdminClient, ident.DID.String()) + if err != nil { + return nil, err + } + ap := AccountPrivate{} + if pv.Email != nil && *pv.Email != "" { + ap.Email = *pv.Email + } + if pv.EmailConfirmedAt != nil && *pv.EmailConfirmedAt != "" { + ap.EmailConfirmed = true + } + ts, err := syntax.ParseDatetimeTime(pv.IndexedAt) + if err != nil { + return nil, err + } + ap.IndexedAt = ts + am.Private = &ap + } + + val, err := json.Marshal(&am) + if err != nil { + return nil, err + } + + if err := e.Cache.Set(ctx, "acct", ident.DID.String(), string(val)); err != nil { + return nil, err + } + return &am, nil +} diff --git a/automod/cachestore.go b/automod/cachestore.go new file mode 100644 index 000000000..4fd33585e --- /dev/null +++ b/automod/cachestore.go @@ -0,0 +1,36 @@ +package automod + +import ( + "context" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +type CacheStore interface { + Get(ctx context.Context, name, key string) (string, error) + Set(ctx context.Context, name, key string, val string) error +} + +type MemCacheStore struct { + Data *expirable.LRU[string, string] +} + +func NewMemCacheStore(capacity int, ttl time.Duration) MemCacheStore { + return MemCacheStore{ + Data: expirable.NewLRU[string, string](capacity, nil, ttl), + } +} + +func (s MemCacheStore) Get(ctx context.Context, name, key string) (string, error) { + v, ok := s.Data.Get(name + "/" + key) + if !ok { + return "", nil + } + return v, nil +} + +func (s MemCacheStore) Set(ctx context.Context, name, key string, val string) error { + s.Data.Add(name+"/"+key, val) + return nil +} diff --git a/automod/countstore.go b/automod/countstore.go new file mode 100644 index 000000000..641594b41 --- /dev/null +++ b/automod/countstore.go @@ -0,0 +1,68 @@ +package automod + +import ( + "context" + "fmt" + "log/slog" + "time" +) + +const ( + PeriodTotal = "total" + PeriodDay = "day" + PeriodHour = "hour" +) + +type CountStore interface { + GetCount(ctx context.Context, name, val, period string) (int, error) + Increment(ctx context.Context, name, val string) error + // TODO: batch increment method +} + +// TODO: this implementation isn't race-safe (yet)! +type MemCountStore struct { + Counts map[string]int +} + +func NewMemCountStore() MemCountStore { + return MemCountStore{ + Counts: make(map[string]int), + } +} + +func PeriodBucket(name, val, period string) string { + switch period { + case PeriodTotal: + return fmt.Sprintf("%s/%s", name, val) + case PeriodDay: + t := time.Now().UTC().Format(time.DateOnly) + return fmt.Sprintf("%s/%s/%s", name, val, t) + case PeriodHour: + t := time.Now().UTC().Format(time.RFC3339)[0:13] + return fmt.Sprintf("%s/%s/%s", name, val, t) + default: + slog.Warn("unhandled counter period", "period", period) + return fmt.Sprintf("%s/%s", name, val) + } +} + +func (s MemCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) { + v, ok := s.Counts[PeriodBucket(name, val, period)] + if !ok { + return 0, nil + } + return v, nil +} + +func (s MemCountStore) Increment(ctx context.Context, name, val string) error { + for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} { + k := PeriodBucket(name, val, p) + v, ok := s.Counts[k] + if !ok { + v = 0 + } + v = v + 1 + s.Counts[k] = v + } + return nil +} diff --git a/automod/doc.go b/automod/doc.go new file mode 100644 index 000000000..e6c025f88 --- /dev/null +++ b/automod/doc.go @@ -0,0 +1,6 @@ +// Auto-Moderation rules engine for anti-spam and other moderation tasks. +// +// This package (`github.com/bluesky-social/indigo/automod`) contains a "rules engine" to augment human moderators in the atproto network. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". A lot of what this package does is collect and maintain caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information. +// +// See `automod/README.md` for more background, and `cmd/hepa` for a daemon built on this package. +package automod diff --git a/automod/engine.go b/automod/engine.go new file mode 100644 index 000000000..1bb2a7cc2 --- /dev/null +++ b/automod/engine.go @@ -0,0 +1,166 @@ +package automod + +import ( + "context" + "fmt" + "log/slog" + "strings" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/xrpc" +) + +// runtime for executing rules, managing state, and recording moderation actions. +// +// TODO: careful when initializing: several fields should not be null or zero, even though they are pointer type. +type Engine struct { + Logger *slog.Logger + Directory identity.Directory + Rules RuleSet + Counters CountStore + Sets SetStore + Cache CacheStore + RelayClient *xrpc.Client + BskyClient *xrpc.Client + // used to persist moderation actions in mod service (optional) + AdminClient *xrpc.Client +} + +func (e *Engine) ProcessIdentityEvent(ctx context.Context, t string, did syntax.DID) error { + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + e.Logger.Error("automod event execution exception", "err", r) + } + }() + + ident, err := e.Directory.LookupDID(ctx, did) + if err != nil { + return fmt.Errorf("resolving identity: %w", err) + } + if ident == nil { + return fmt.Errorf("identity not found for did: %s", did.String()) + } + + am, err := e.GetAccountMeta(ctx, ident) + if err != nil { + return err + } + evt := IdentityEvent{ + RepoEvent{ + Engine: e, + Logger: e.Logger.With("did", am.Identity.DID), + Account: *am, + }, + } + if err := e.Rules.CallIdentityRules(&evt); err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + evt.CanonicalLogLine() + if err := evt.PersistActions(ctx); err != nil { + return err + } + return nil +} + +func (e *Engine) ProcessRecord(ctx context.Context, did syntax.DID, path, recCID string, rec any) error { + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + e.Logger.Error("automod event execution exception", "err", r) + } + }() + + ident, err := e.Directory.LookupDID(ctx, did) + if err != nil { + return fmt.Errorf("resolving identity: %w", err) + } + if ident == nil { + return fmt.Errorf("identity not found for did: %s", did.String()) + } + + am, err := e.GetAccountMeta(ctx, ident) + if err != nil { + return err + } + evt := e.NewRecordEvent(*am, path, recCID, rec) + e.Logger.Debug("processing record", "did", ident.DID, "path", path) + if err := e.Rules.CallRecordRules(&evt); err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + evt.CanonicalLogLine() + if err := evt.PersistActions(ctx); err != nil { + return err + } + if err := evt.PersistCounters(ctx); err != nil { + return err + } + + return nil +} + +func (e *Engine) FetchAndProcessRecord(ctx context.Context, uri string) error { + // resolve URI, identity, and record + aturi, err := syntax.ParseATURI(uri) + if err != nil { + return fmt.Errorf("parsing AT-URI argument: %v", err) + } + if aturi.RecordKey() == "" { + return fmt.Errorf("need a full, not partial, AT-URI: %s", uri) + } + ident, err := e.Directory.Lookup(ctx, aturi.Authority()) + if err != nil { + return fmt.Errorf("resolving AT-URI authority: %v", err) + } + pdsURL := ident.PDSEndpoint() + if pdsURL == "" { + return fmt.Errorf("could not resolve PDS endpoint for AT-URI account: %s", ident.DID.String()) + } + pdsClient := xrpc.Client{Host: ident.PDSEndpoint()} + + e.Logger.Info("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + out, err := comatproto.RepoGetRecord(ctx, &pdsClient, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String()) + if err != nil { + return fmt.Errorf("fetching record from Relay (%s): %v", aturi, err) + } + if out.Cid == nil { + return fmt.Errorf("expected a CID in getRecord response") + } + return e.ProcessRecord(ctx, ident.DID, aturi.Path(), *out.Cid, out.Value.Val) +} + +func (e *Engine) NewRecordEvent(am AccountMeta, path, recCID string, rec any) RecordEvent { + parts := strings.SplitN(path, "/", 2) + return RecordEvent{ + RepoEvent{ + Engine: e, + Logger: e.Logger.With("did", am.Identity.DID, "collection", parts[0], "rkey", parts[1]), + Account: am, + }, + rec, + parts[0], + parts[1], + recCID, + []string{}, + false, + []ModReport{}, + []string{}, + } +} + +func (e *Engine) GetCount(name, val, period string) (int, error) { + return e.Counters.GetCount(context.TODO(), name, val, period) +} + +// checks if `val` is an element of set `name` +func (e *Engine) InSet(name, val string) (bool, error) { + return e.Sets.InSet(context.TODO(), name, val) +} diff --git a/automod/engine_test.go b/automod/engine_test.go new file mode 100644 index 000000000..a597f2c9c --- /dev/null +++ b/automod/engine_test.go @@ -0,0 +1,82 @@ +package automod + +import ( + "context" + "log/slog" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func simpleRule(evt *RecordEvent, post *appbsky.FeedPost) error { + for _, tag := range post.Tags { + if evt.InSet("banned-hashtags", tag) { + evt.AddRecordLabel("bad-hashtag") + break + } + } + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if feat.RichtextFacet_Tag != nil { + tag := feat.RichtextFacet_Tag.Tag + if evt.InSet("banned-hashtags", tag) { + evt.AddRecordLabel("bad-hashtag") + break + } + } + } + } + return nil +} + +func engineFixture() Engine { + rules := RuleSet{ + PostRules: []PostRuleFunc{ + simpleRule, + }, + } + sets := NewMemSetStore() + sets.Sets["banned-hashtags"] = make(map[string]bool) + sets.Sets["banned-hashtags"]["slur"] = true + dir := identity.NewMockDirectory() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + dir.Insert(id1) + engine := Engine{ + Logger: slog.Default(), + Directory: &dir, + Counters: NewMemCountStore(), + Sets: sets, + Rules: rules, + } + return engine +} + +func TestEngineBasics(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + engine := engineFixture() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + path := "app.bsky.feed.post/abc123" + cid1 := "cid123" + p1 := appbsky.FeedPost{ + Text: "some post blah", + } + assert.NoError(engine.ProcessRecord(ctx, id1.DID, path, cid1, &p1)) + + p2 := appbsky.FeedPost{ + Text: "some post blah", + Tags: []string{"one", "slur"}, + } + assert.NoError(engine.ProcessRecord(ctx, id1.DID, path, cid1, &p2)) +} diff --git a/automod/event.go b/automod/event.go new file mode 100644 index 000000000..2de194eae --- /dev/null +++ b/automod/event.go @@ -0,0 +1,263 @@ +package automod + +import ( + "context" + "fmt" + "log/slog" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +type ModReport struct { + ReasonType string + Comment string +} + +type CounterRef struct { + Name string + Val string +} + +// base type for events specific to an account, usually derived from a repo event stream message (one such message may result in multiple `RepoEvent`) +// +// events are both containers for data about the event itself (similar to an HTTP request type); aggregate results and state (counters, mod actions) to be persisted after all rules are run; and act as an API for additional network reads and operations. +type RepoEvent struct { + Engine *Engine + Err error + Logger *slog.Logger + Account AccountMeta + CounterIncrements []CounterRef + AccountLabels []string + AccountFlags []string + AccountReports []ModReport + AccountTakedown bool +} + +func (e *RepoEvent) GetCount(name, val, period string) int { + v, err := e.Engine.GetCount(name, val, period) + if err != nil { + e.Err = err + return 0 + } + return v +} + +func (e *RepoEvent) InSet(name, val string) bool { + v, err := e.Engine.InSet(name, val) + if err != nil { + e.Err = err + return false + } + return v +} + +func (e *RepoEvent) Increment(name, val string) { + e.CounterIncrements = append(e.CounterIncrements, CounterRef{Name: name, Val: val}) +} + +func (e *RepoEvent) TakedownAccount() { + e.AccountTakedown = true +} + +func (e *RepoEvent) AddAccountLabel(val string) { + e.AccountLabels = append(e.AccountLabels, val) +} + +func (e *RepoEvent) AddAccountFlag(val string) { + e.AccountFlags = append(e.AccountFlags, val) +} + +func (e *RepoEvent) ReportAccount(reason, comment string) { + e.AccountReports = append(e.AccountReports, ModReport{ReasonType: reason, Comment: comment}) +} + +func (e *RepoEvent) PersistAccountActions(ctx context.Context) error { + if e.Engine.AdminClient == nil { + return nil + } + xrpcc := e.Engine.AdminClient + if len(e.AccountLabels) > 0 { + _, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{ + Action: "com.atproto.admin.defs#flag", + CreateLabelVals: dedupeStrings(e.AccountLabels), + Reason: "automod", + CreatedBy: xrpcc.Auth.Did, + Subject: &comatproto.AdminTakeModerationAction_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: e.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + return err + } + } + // TODO: AccountFlags + for _, mr := range e.AccountReports { + _, err := comatproto.ModerationCreateReport(ctx, xrpcc, &comatproto.ModerationCreateReport_Input{ + ReasonType: &mr.ReasonType, + Reason: &mr.Comment, + Subject: &comatproto.ModerationCreateReport_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: e.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + return err + } + } + if e.AccountTakedown { + _, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{ + Action: "com.atproto.admin.defs#takedown", + Reason: "automod", + CreatedBy: xrpcc.Auth.Did, + Subject: &comatproto.AdminTakeModerationAction_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: e.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + return err + } + } + return nil +} + +func (e *RepoEvent) PersistActions(ctx context.Context) error { + return e.PersistAccountActions(ctx) +} + +func (e *RepoEvent) PersistCounters(ctx context.Context) error { + // TODO: dedupe this array + for _, ref := range e.CounterIncrements { + err := e.Engine.Counters.Increment(ctx, ref.Name, ref.Val) + if err != nil { + return err + } + } + return nil +} + +func (e *RepoEvent) CanonicalLogLine() { + e.Logger.Info("canonical-event-line", + "accountLabels", e.AccountLabels, + "accountFlags", e.AccountFlags, + "accountTakedown", e.AccountTakedown, + "accountReports", len(e.AccountReports), + ) +} + +type IdentityEvent struct { + RepoEvent +} + +type RecordEvent struct { + RepoEvent + + Record any + Collection string + RecordKey string + CID string + RecordLabels []string + RecordTakedown bool + RecordReports []ModReport + RecordFlags []string + // TODO: commit metadata +} + +func (e *RecordEvent) TakedownRecord() { + e.RecordTakedown = true +} + +func (e *RecordEvent) AddRecordLabel(val string) { + e.RecordLabels = append(e.RecordLabels, val) +} + +func (e *RecordEvent) AddRecordFlag(val string) { + e.RecordFlags = append(e.RecordFlags, val) +} + +func (e *RecordEvent) ReportRecord(reason, comment string) { + e.RecordReports = append(e.RecordReports, ModReport{ReasonType: reason, Comment: comment}) +} + +func (e *RecordEvent) PersistRecordActions(ctx context.Context) error { + if e.Engine.AdminClient == nil { + return nil + } + strongRef := comatproto.RepoStrongRef{ + Cid: e.CID, + Uri: fmt.Sprintf("at://%s/%s/%s", e.Account.Identity.DID, e.Collection, e.RecordKey), + } + xrpcc := e.Engine.AdminClient + if len(e.RecordLabels) > 0 { + // TODO: this does an action, not just create labels; will update after event refactor + _, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{ + Action: "com.atproto.admin.defs#flag", + CreateLabelVals: dedupeStrings(e.RecordLabels), + Reason: "automod", + CreatedBy: xrpcc.Auth.Did, + Subject: &comatproto.AdminTakeModerationAction_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + return err + } + } + // TODO: AccountFlags + for _, mr := range e.RecordReports { + _, err := comatproto.ModerationCreateReport(ctx, xrpcc, &comatproto.ModerationCreateReport_Input{ + ReasonType: &mr.ReasonType, + Reason: &mr.Comment, + Subject: &comatproto.ModerationCreateReport_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + return err + } + } + if e.RecordTakedown { + _, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{ + Action: "com.atproto.admin.defs#takedown", + Reason: "automod", + CreatedBy: xrpcc.Auth.Did, + Subject: &comatproto.AdminTakeModerationAction_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + return err + } + } + return nil +} + +func (e *RecordEvent) PersistActions(ctx context.Context) error { + if err := e.PersistAccountActions(ctx); err != nil { + return err + } + return e.PersistRecordActions(ctx) +} + +func (e *RecordEvent) CanonicalLogLine() { + e.Logger.Info("canonical-event-line", + "accountLabels", e.AccountLabels, + "accountFlags", e.AccountFlags, + "accountTakedown", e.AccountTakedown, + "accountReports", len(e.AccountReports), + "recordLabels", e.RecordLabels, + "recordFlags", e.RecordFlags, + "recordTakedown", e.RecordTakedown, + "recordReports", len(e.RecordReports), + ) +} + +type IdentityRuleFunc = func(evt *IdentityEvent) error +type RecordRuleFunc = func(evt *RecordEvent) error +type PostRuleFunc = func(evt *RecordEvent, post *appbsky.FeedPost) error +type ProfileRuleFunc = func(evt *RecordEvent, profile *appbsky.ActorProfile) error diff --git a/automod/redis_cache.go b/automod/redis_cache.go new file mode 100644 index 000000000..c5724826d --- /dev/null +++ b/automod/redis_cache.go @@ -0,0 +1,63 @@ +package automod + +import ( + "context" + "time" + + "github.com/go-redis/cache/v9" + "github.com/redis/go-redis/v9" +) + +type RedisCacheStore struct { + Data *cache.Cache + TTL time.Duration +} + +var _ CacheStore = (*RedisCacheStore)(nil) + +func NewRedisCacheStore(redisURL string, ttl time.Duration) (*RedisCacheStore, error) { + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, err + } + data := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(10_000, ttl), + }) + return &RedisCacheStore{ + Data: data, + TTL: ttl, + }, nil +} + +func redisCacheKey(name, key string) string { + return "cache/" + name + "/" + key +} + +func (s RedisCacheStore) Get(ctx context.Context, name, key string) (string, error) { + var val string + err := s.Data.Get(ctx, redisCacheKey(name, key), &val) + if err == cache.ErrCacheMiss { + return "", nil + } + if err != nil { + return "", err + } + return val, nil +} + +func (s RedisCacheStore) Set(ctx context.Context, name, key string, val string) error { + s.Data.Set(&cache.Item{ + Ctx: ctx, + Key: redisCacheKey(name, key), + Value: val, + TTL: s.TTL, + }) + return nil +} diff --git a/automod/redis_counters.go b/automod/redis_counters.go new file mode 100644 index 000000000..f95fdbd8c --- /dev/null +++ b/automod/redis_counters.go @@ -0,0 +1,65 @@ +package automod + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +var redisCountPrefix string = "count/" + +type RedisCountStore struct { + Client *redis.Client +} + +func NewRedisCountStore(redisURL string) (*RedisCountStore, error) { + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, err + } + rcs := RedisCountStore{ + Client: rdb, + } + return &rcs, nil +} + +func (s *RedisCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) { + key := redisCountPrefix + PeriodBucket(name, val, period) + c, err := s.Client.Get(ctx, key).Int() + if err == redis.Nil { + return 0, nil + } else if err != nil { + return 0, err + } + return c, nil +} + +func (s *RedisCountStore) Increment(ctx context.Context, name, val string) error { + + var key string + + // increment multiple counters in a single redis round-trip + multi := s.Client.Pipeline() + + key = redisCountPrefix + PeriodBucket(name, val, PeriodHour) + multi.Incr(ctx, key) + multi.Expire(ctx, key, 2*time.Hour) + + key = redisCountPrefix + PeriodBucket(name, val, PeriodDay) + multi.Incr(ctx, key) + multi.Expire(ctx, key, 48*time.Hour) + + key = redisCountPrefix + PeriodBucket(name, val, PeriodTotal) + multi.Incr(ctx, key) + // no expiration for total + + _, err := multi.Exec(ctx) + return err +} diff --git a/automod/redis_directory.go b/automod/redis_directory.go new file mode 100644 index 000000000..4a76c8b4d --- /dev/null +++ b/automod/redis_directory.go @@ -0,0 +1,354 @@ +package automod + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/go-redis/cache/v9" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/redis/go-redis/v9" +) + +var redisDirPrefix string = "dir/" + +// uses redis as a cache for identity lookups. includes a local cache layer as well, for hot keys +type RedisDirectory struct { + Inner identity.Directory + ErrTTL time.Duration + HitTTL time.Duration + + handleCache *cache.Cache + identityCache *cache.Cache + didLookupChans sync.Map + handleLookupChans sync.Map +} + +type HandleEntry struct { + Updated time.Time + DID syntax.DID + Err error +} + +type IdentityEntry struct { + Updated time.Time + Identity *identity.Identity + Err error +} + +var _ identity.Directory = (*RedisDirectory)(nil) + +func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL time.Duration) (*RedisDirectory, error) { + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, err + } + handleCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(10_000, hitTTL), + }) + identityCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(10_000, hitTTL), + }) + return &RedisDirectory{ + Inner: inner, + ErrTTL: errTTL, + HitTTL: hitTTL, + handleCache: handleCache, + identityCache: identityCache, + }, nil +} + +func (d *RedisDirectory) IsHandleStale(e *HandleEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *RedisDirectory) IsIdentityStale(e *IdentityEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*HandleEntry, error) { + ident, err := d.Inner.LookupHandle(ctx, h) + if err != nil { + he := HandleEntry{ + Updated: time.Now(), + DID: "", + Err: err, + } + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + h.String(), + Value: he, + TTL: d.ErrTTL, + }) + if err != nil { + return nil, err + } + return &he, nil + } + + ident.ParsedPublicKey = nil + entry := IdentityEntry{ + Updated: time.Now(), + Identity: ident, + Err: nil, + } + he := HandleEntry{ + Updated: time.Now(), + DID: ident.DID, + Err: nil, + } + + err = d.identityCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + ident.DID.String(), + Value: entry, + TTL: d.HitTTL, + }) + if err != nil { + return nil, err + } + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + h.String(), + Value: he, + TTL: d.HitTTL, + }) + if err != nil { + return nil, err + } + return &he, nil +} + +func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + var entry HandleEntry + err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return "", err + } + if err != cache.ErrCacheMiss && !d.IsHandleStale(&entry) { + handleCacheHits.Inc() + return entry.DID, entry.Err + } + handleCacheMisses.Inc() + + // Coalesce multiple requests for the same Handle + res := make(chan struct{}) + val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res) + if loaded { + handleRequestsCoalesced.Inc() + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), entry) + if err != nil && err != cache.ErrCacheMiss { + return "", err + } + if err != cache.ErrCacheMiss && !d.IsHandleStale(&entry) { + return entry.DID, entry.Err + } + return "", fmt.Errorf("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return "", ctx.Err() + } + } + + var did syntax.DID + // Update the Handle Entry from PLC and cache the result + newEntry, err := d.updateHandle(ctx, h) + if err == nil && newEntry != nil { + did = newEntry.DID + } + // Cleanup the coalesce map and close the results channel + d.handleLookupChans.Delete(h.String()) + // Callers waiting will now get the result from the cache + close(res) + + return did, err +} + +func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*IdentityEntry, error) { + ident, err := d.Inner.LookupDID(ctx, did) + // wipe parsed public key; it's a waste of space and can't serialize + if nil == err { + ident.ParsedPublicKey = nil + } + // persist the identity lookup error, instead of processing it immediately + entry := IdentityEntry{ + Updated: time.Now(), + Identity: ident, + Err: err, + } + var he *HandleEntry + // if *not* an error, then also update the handle cache + if nil == err && !ident.Handle.IsInvalidHandle() { + he = &HandleEntry{ + Updated: time.Now(), + DID: did, + Err: nil, + } + } + + err = d.identityCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + did.String(), + Value: entry, + TTL: d.HitTTL, + }) + if err != nil { + return nil, err + } + if he != nil { + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + ident.Handle.String(), + Value: *he, + TTL: d.HitTTL, + }) + if err != nil { + return nil, err + } + } + return &entry, nil +} + +func (d *RedisDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identity.Identity, error) { + var entry IdentityEntry + err := d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return nil, err + } + if err != cache.ErrCacheMiss && !d.IsIdentityStale(&entry) { + identityCacheHits.Inc() + return entry.Identity, entry.Err + } + identityCacheMisses.Inc() + + // Coalesce multiple requests for the same DID + res := make(chan struct{}) + val, loaded := d.didLookupChans.LoadOrStore(did.String(), res) + if loaded { + identityRequestsCoalesced.Inc() + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err = d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return nil, err + } + if err != cache.ErrCacheMiss && !d.IsIdentityStale(&entry) { + return entry.Identity, entry.Err + } + return nil, fmt.Errorf("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + var doc *identity.Identity + // Update the Identity Entry from PLC and cache the result + newEntry, err := d.updateDID(ctx, did) + if err == nil && newEntry != nil { + doc = newEntry.Identity + } + // Cleanup the coalesce map and close the results channel + d.didLookupChans.Delete(did.String()) + // Callers waiting will now get the result from the cache + close(res) + + return doc, err +} + +func (d *RedisDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*identity.Identity, error) { + did, err := d.ResolveHandle(ctx, h) + if err != nil { + return nil, err + } + ident, err := d.LookupDID(ctx, did) + if err != nil { + return nil, err + } + + declared, err := ident.DeclaredHandle() + if err != nil { + return nil, err + } + if declared != h { + return nil, fmt.Errorf("handle does not match that declared in DID document") + } + return ident, nil +} + +func (d *RedisDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*identity.Identity, error) { + handle, err := a.AsHandle() + if nil == err { // if not an error, is a handle + return d.LookupHandle(ctx, handle) + } + did, err := a.AsDID() + if nil == err { // if not an error, is a DID + return d.LookupDID(ctx, did) + } + return nil, fmt.Errorf("at-identifier neither a Handle nor a DID") +} + +func (d *RedisDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error { + handle, err := a.AsHandle() + if nil == err { // if not an error, is a handle + return d.handleCache.Delete(ctx, handle.String()) + } + did, err := a.AsDID() + if nil == err { // if not an error, is a DID + return d.identityCache.Delete(ctx, did.String()) + } + return fmt.Errorf("at-identifier neither a Handle nor a DID") +} + +var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_cache_hits", + Help: "Number of cache hits for ATProto handle lookups", +}) + +var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_cache_misses", + Help: "Number of cache misses for ATProto handle lookups", +}) + +var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_cache_hits", + Help: "Number of cache hits for ATProto identity lookups", +}) + +var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_cache_misses", + Help: "Number of cache misses for ATProto identity lookups", +}) + +var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_requests_coalesced", + Help: "Number of identity requests coalesced", +}) + +var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_requests_coalesced", + Help: "Number of handle requests coalesced", +}) diff --git a/automod/rules/all.go b/automod/rules/all.go new file mode 100644 index 000000000..86f4b6bdf --- /dev/null +++ b/automod/rules/all.go @@ -0,0 +1,19 @@ +package rules + +import ( + "github.com/bluesky-social/indigo/automod" +) + +func DefaultRules() automod.RuleSet { + rules := automod.RuleSet{ + PostRules: []automod.PostRuleFunc{ + MisleadingURLPostRule, + MisleadingMentionPostRule, + ReplyCountPostRule, + BanHashtagsPostRule, + AccountDemoPostRule, + AccountPrivateDemoPostRule, + }, + } + return rules +} diff --git a/automod/rules/example_sets.json b/automod/rules/example_sets.json new file mode 100644 index 000000000..d66de7831 --- /dev/null +++ b/automod/rules/example_sets.json @@ -0,0 +1,6 @@ +{ + "banned-hashtags": [ + "slur", + "anotherslur" + ] +} diff --git a/automod/rules/fixture_test.go b/automod/rules/fixture_test.go new file mode 100644 index 000000000..6328c4702 --- /dev/null +++ b/automod/rules/fixture_test.go @@ -0,0 +1,44 @@ +package rules + +import ( + "log/slog" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/xrpc" +) + +func engineFixture() automod.Engine { + rules := automod.RuleSet{ + PostRules: []automod.PostRuleFunc{ + BanHashtagsPostRule, + }, + } + sets := automod.NewMemSetStore() + sets.Sets["banned-hashtags"] = make(map[string]bool) + sets.Sets["banned-hashtags"]["slur"] = true + dir := identity.NewMockDirectory() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + id2 := identity.Identity{ + DID: syntax.DID("did:plc:abc222"), + Handle: syntax.Handle("imposter.example.com"), + } + dir.Insert(id1) + dir.Insert(id2) + adminc := xrpc.Client{ + Host: "http://dummy.local", + } + engine := automod.Engine{ + Logger: slog.Default(), + Directory: &dir, + Counters: automod.NewMemCountStore(), + Sets: sets, + Rules: rules, + AdminClient: &adminc, + } + return engine +} diff --git a/automod/rules/hashtags.go b/automod/rules/hashtags.go new file mode 100644 index 000000000..aa047a207 --- /dev/null +++ b/automod/rules/hashtags.go @@ -0,0 +1,16 @@ +package rules + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +func BanHashtagsPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + for _, tag := range ExtractHashtags(post) { + if evt.InSet("banned-hashtags", tag) { + evt.AddRecordFlag("bad-hashtag") + break + } + } + return nil +} diff --git a/automod/rules/hashtags_test.go b/automod/rules/hashtags_test.go new file mode 100644 index 000000000..678aa0299 --- /dev/null +++ b/automod/rules/hashtags_test.go @@ -0,0 +1,40 @@ +package rules + +import ( + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + + "github.com/stretchr/testify/assert" +) + +func TestBanHashtagPostRule(t *testing.T) { + assert := assert.New(t) + + engine := engineFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + path := "app.bsky.feed.post/abc123" + cid1 := "cid123" + p1 := appbsky.FeedPost{ + Text: "some post blah", + } + evt1 := engine.NewRecordEvent(am1, path, cid1, &p1) + assert.NoError(BanHashtagsPostRule(&evt1, &p1)) + assert.Empty(evt1.RecordFlags) + + p2 := appbsky.FeedPost{ + Text: "some post blah", + Tags: []string{"one", "slur"}, + } + evt2 := engine.NewRecordEvent(am1, path, cid1, &p2) + assert.NoError(BanHashtagsPostRule(&evt2, &p2)) + assert.NotEmpty(evt2.RecordFlags) +} diff --git a/automod/rules/helpers.go b/automod/rules/helpers.go new file mode 100644 index 000000000..d702ef059 --- /dev/null +++ b/automod/rules/helpers.go @@ -0,0 +1,78 @@ +package rules + +import ( + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +func dedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} + +func ExtractHashtags(post *appbsky.FeedPost) []string { + var tags []string + for _, tag := range post.Tags { + tags = append(tags, tag) + } + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if feat.RichtextFacet_Tag != nil { + tags = append(tags, feat.RichtextFacet_Tag.Tag) + } + } + } + return dedupeStrings(tags) +} + +type PostFacet struct { + Text string + URL *string + DID *string + Tag *string +} + +func ExtractFacets(post *appbsky.FeedPost) ([]PostFacet, error) { + var out []PostFacet + + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if int(facet.Index.ByteEnd) > len([]byte(post.Text)) || facet.Index.ByteStart > facet.Index.ByteEnd { + return nil, fmt.Errorf("invalid facet byte range") + } + + txt := string([]byte(post.Text)[facet.Index.ByteStart:facet.Index.ByteEnd]) + if txt == "" { + return nil, fmt.Errorf("empty facet text") + } + + if feat.RichtextFacet_Link != nil { + out = append(out, PostFacet{ + Text: txt, + URL: &feat.RichtextFacet_Link.Uri, + }) + } + if feat.RichtextFacet_Tag != nil { + out = append(out, PostFacet{ + Text: txt, + Tag: &feat.RichtextFacet_Tag.Tag, + }) + } + if feat.RichtextFacet_Mention != nil { + out = append(out, PostFacet{ + Text: txt, + DID: &feat.RichtextFacet_Mention.Did, + }) + } + } + } + return out, nil +} diff --git a/automod/rules/misleading.go b/automod/rules/misleading.go new file mode 100644 index 000000000..b8668ead2 --- /dev/null +++ b/automod/rules/misleading.go @@ -0,0 +1,94 @@ +package rules + +import ( + "context" + "net/url" + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" +) + +func MisleadingURLPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + facets, err := ExtractFacets(post) + if err != nil { + evt.Logger.Warn("invalid facets", "err", err) + evt.AddRecordFlag("invalid") // TODO: or some other "this record is corrupt" indicator? + return nil + } + for _, facet := range facets { + if facet.URL != nil { + linkURL, err := url.Parse(*facet.URL) + if err != nil { + evt.Logger.Warn("invalid link metadata URL", "url", facet.URL) + continue + } + + // basic text string pre-cleanups + text := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(facet.Text), "...")) + // if really not a domain, just skipp + if !strings.Contains(text, ".") { + continue + } + // try to fix any missing method in the text + if !strings.Contains(text, "://") { + text = "https://" + text + } + + // try parsing as a full URL (with whitespace trimmed) + textURL, err := url.Parse(text) + if err != nil { + evt.Logger.Warn("invalid link text URL", "url", facet.Text) + continue + } + + // for now just compare domains to handle the most obvious cases + // this public code will obviously get discovered and bypassed. this doesn't earn you any security cred! + if linkURL.Host != textURL.Host && linkURL.Host != "www."+linkURL.Host { + evt.Logger.Warn("misleading mismatched domains", "linkHost", linkURL.Host, "textHost", textURL.Host, "text", facet.Text) + evt.AddRecordFlag("misleading") + } + } + } + return nil +} + +func MisleadingMentionPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + // TODO: do we really need to route context around? probably + ctx := context.TODO() + facets, err := ExtractFacets(post) + if err != nil { + evt.Logger.Warn("invalid facets", "err", err) + evt.AddRecordFlag("invalid") // TODO: or some other "this record is corrupt" indicator? + return nil + } + for _, facet := range facets { + if facet.DID != nil { + txt := facet.Text + if txt[0] == '@' { + txt = txt[1:] + } + handle, err := syntax.ParseHandle(txt) + if err != nil { + evt.Logger.Warn("mention was not a valid handle", "text", txt) + continue + } + + mentioned, err := evt.Engine.Directory.LookupHandle(ctx, handle) + if err != nil { + evt.Logger.Warn("could not resolve handle", "handle", handle) + evt.AddRecordFlag("misleading") + break + } + + // TODO: check if mentioned DID was recently updated? might be a caching issue + if mentioned.DID.String() != *facet.DID { + evt.Logger.Warn("misleading mention", "text", txt, "did", facet.DID) + evt.AddRecordFlag("misleading") + continue + } + } + } + return nil +} diff --git a/automod/rules/misleading_test.go b/automod/rules/misleading_test.go new file mode 100644 index 000000000..fee4444ec --- /dev/null +++ b/automod/rules/misleading_test.go @@ -0,0 +1,82 @@ +package rules + +import ( + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + + "github.com/stretchr/testify/assert" +) + +func TestMisleadingURLPostRule(t *testing.T) { + assert := assert.New(t) + + engine := engineFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + path := "app.bsky.feed.post/abc123" + cid1 := "cid123" + p1 := appbsky.FeedPost{ + Text: "https://safe.com/ is very reputable", + Facets: []*appbsky.RichtextFacet{ + &appbsky.RichtextFacet{ + Features: []*appbsky.RichtextFacet_Features_Elem{ + &appbsky.RichtextFacet_Features_Elem{ + RichtextFacet_Link: &appbsky.RichtextFacet_Link{ + Uri: "https://evil.com", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 0, + ByteEnd: 16, + }, + }, + }, + } + evt1 := engine.NewRecordEvent(am1, path, cid1, &p1) + assert.NoError(MisleadingURLPostRule(&evt1, &p1)) + assert.NotEmpty(evt1.RecordFlags) +} + +func TestMisleadingMentionPostRule(t *testing.T) { + assert := assert.New(t) + + engine := engineFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + path := "app.bsky.feed.post/abc123" + cid1 := "cid123" + p1 := appbsky.FeedPost{ + Text: "@handle.example.com is a friend", + Facets: []*appbsky.RichtextFacet{ + &appbsky.RichtextFacet{ + Features: []*appbsky.RichtextFacet_Features_Elem{ + &appbsky.RichtextFacet_Features_Elem{ + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc222", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 1, + ByteEnd: 19, + }, + }, + }, + } + evt1 := engine.NewRecordEvent(am1, path, cid1, &p1) + assert.NoError(MisleadingMentionPostRule(&evt1, &p1)) + assert.NotEmpty(evt1.RecordFlags) +} diff --git a/automod/rules/private.go b/automod/rules/private.go new file mode 100644 index 000000000..6382dfe24 --- /dev/null +++ b/automod/rules/private.go @@ -0,0 +1,18 @@ +package rules + +import ( + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +// dummy rule. this leaks PII (account email) in logs and should never be used in real life +func AccountPrivateDemoPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + if evt.Account.Private != nil { + if strings.HasSuffix(evt.Account.Private.Email, "@blueskyweb.xyz") { + evt.Logger.Info("hello dev!", "email", evt.Account.Private.Email) + } + } + return nil +} diff --git a/automod/rules/profile.go b/automod/rules/profile.go new file mode 100644 index 000000000..5b69e55f0 --- /dev/null +++ b/automod/rules/profile.go @@ -0,0 +1,14 @@ +package rules + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +// this is a dummy rule to demonstrate accessing account metadata (eg, profile) from within post handler +func AccountDemoPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + if evt.Account.Profile.Description != nil && len(post.Text) > 5 && *evt.Account.Profile.Description == post.Text { + evt.AddRecordFlag("own-profile-description") + } + return nil +} diff --git a/automod/rules/replies.go b/automod/rules/replies.go new file mode 100644 index 000000000..db086cba2 --- /dev/null +++ b/automod/rules/replies.go @@ -0,0 +1,17 @@ +package rules + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +func ReplyCountPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error { + if post.Reply != nil { + did := evt.Account.Identity.DID.String() + if evt.GetCount("reply", did, automod.PeriodDay) > 3 { + evt.AddAccountFlag("frequent-replier") + } + evt.Increment("reply", did) + } + return nil +} diff --git a/automod/ruleset.go b/automod/ruleset.go new file mode 100644 index 000000000..12d0794c4 --- /dev/null +++ b/automod/ruleset.go @@ -0,0 +1,72 @@ +package automod + +import ( + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +type RuleSet struct { + PostRules []PostRuleFunc + ProfileRules []ProfileRuleFunc + RecordRules []RecordRuleFunc + IdentityRules []IdentityRuleFunc +} + +func (r *RuleSet) CallRecordRules(evt *RecordEvent) error { + // first the generic rules + for _, f := range r.RecordRules { + err := f(evt) + if err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + } + // then any record-type-specific rules + switch evt.Collection { + case "app.bsky.feed.post": + post, ok := evt.Record.(*appbsky.FeedPost) + if !ok { + return fmt.Errorf("mismatch between collection (%s) and type", evt.Collection) + } + for _, f := range r.PostRules { + err := f(evt, post) + if err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + } + case "app.bsky.actor.profile": + profile, ok := evt.Record.(*appbsky.ActorProfile) + if !ok { + return fmt.Errorf("mismatch between collection (%s) and type", evt.Collection) + } + for _, f := range r.ProfileRules { + err := f(evt, profile) + if err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + } + } + return nil +} + +func (r *RuleSet) CallIdentityRules(evt *IdentityEvent) error { + for _, f := range r.IdentityRules { + err := f(evt) + if err != nil { + return err + } + if evt.Err != nil { + return evt.Err + } + } + return nil +} diff --git a/automod/setstore.go b/automod/setstore.go new file mode 100644 index 000000000..29acddbc7 --- /dev/null +++ b/automod/setstore.go @@ -0,0 +1,61 @@ +package automod + +import ( + "context" + "encoding/json" + "io" + "os" +) + +type SetStore interface { + InSet(ctx context.Context, name, val string) (bool, error) +} + +// TODO: this implementation isn't race-safe (yet)! +type MemSetStore struct { + Sets map[string]map[string]bool +} + +func NewMemSetStore() MemSetStore { + return MemSetStore{ + Sets: make(map[string]map[string]bool), + } +} + +func (s MemSetStore) InSet(ctx context.Context, name, val string) (bool, error) { + set, ok := s.Sets[name] + if !ok { + // NOTE: currently returns false when entire set isn't found + return false, nil + } + _, ok = set[val] + return ok, nil +} + +func (s *MemSetStore) LoadFromFileJSON(p string) error { + + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + raw, err := io.ReadAll(f) + if err != nil { + return err + } + + var rules map[string][]string + if err := json.Unmarshal(raw, &rules); err != nil { + return err + } + + for name, l := range rules { + m := make(map[string]bool, len(l)) + for _, val := range l { + m[val] = true + } + s.Sets[name] = m + } + return nil +} diff --git a/automod/util.go b/automod/util.go new file mode 100644 index 000000000..6934fe54b --- /dev/null +++ b/automod/util.go @@ -0,0 +1,13 @@ +package automod + +func dedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} diff --git a/cmd/hepa/Dockerfile b/cmd/hepa/Dockerfile new file mode 100644 index 000000000..cfee95d3b --- /dev/null +++ b/cmd/hepa/Dockerfile @@ -0,0 +1,37 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/hepa/Dockerfile -t hepa . + +### Compile stage +FROM golang:1.21-alpine3.18 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /hepa ./cmd/hepa + +### Run stage +FROM alpine:3.18 + +RUN apk add --no-cache --update dumb-init ca-certificates +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/hepa +COPY --from=build-env /hepa / + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 2210 + +CMD ["/hepa"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="ATP Auto-Moderation Service (hepa)" +LABEL org.opencontainers.image.licenses=MIT diff --git a/cmd/hepa/README.md b/cmd/hepa/README.md new file mode 100644 index 000000000..766952ade --- /dev/null +++ b/cmd/hepa/README.md @@ -0,0 +1,22 @@ + +hepa +==== + +This is a simple auto-moderation daemon which wraps the automod package. + +The name is a reference to HEPA air filters, which help keep the local atmosphere clean and healthy for humans. + +Available commands, flags, and config are documented in the usage (`--help`). + +Current features and design decisions: + +- all state (counters) and caches stored in Redis +- consumes from Relay firehose; no backfill functionality yet +- which rules are included configured at compile time +- admin access to fetch private account metadata, and to persist moderation actions, is optional. it is possible for anybody to run a `hepa` instance + +This is not a "labeling service" per say, in that it pushes labels in to an existing moderation service, and doesn't provide API endpoints or label streams. see `labelmaker` for a self-contained labeling service. + +Performance is generally slow when first starting up, because account-level metadata is being fetched (and cached) for every firehose event. After the caches have "warmed up", events are processed faster. + +See the `automod` package's README for more documentation. diff --git a/cmd/hepa/consumer.go b/cmd/hepa/consumer.go new file mode 100644 index 000000000..b9b1434f4 --- /dev/null +++ b/cmd/hepa/consumer.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/events/schedulers/autoscaling" + lexutil "github.com/bluesky-social/indigo/lex/util" + + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/repomgr" + "github.com/carlmjohnson/versioninfo" + "github.com/gorilla/websocket" +) + +func (s *Server) RunConsumer(ctx context.Context) error { + + // TODO: persist cursor in a database or local disk + cur, err := s.ReadLastCursor(ctx) + if err != nil { + return err + } + + dialer := websocket.DefaultDialer + u, err := url.Parse(s.bgshost) + if err != nil { + return fmt.Errorf("invalid bgshost URI: %w", err) + } + u.Path = "xrpc/com.atproto.sync.subscribeRepos" + if cur != 0 { + u.RawQuery = fmt.Sprintf("cursor=%d", cur) + } + s.logger.Info("subscribing to repo event stream", "upstream", s.bgshost, "cursor", cur) + con, _, err := dialer.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("hepa/%s", versioninfo.Short())}, + }) + if err != nil { + return fmt.Errorf("subscribing to firehose failed (dialing): %w", err) + } + + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + s.lastSeq = evt.Seq + return s.HandleRepoCommit(ctx, evt) + }, + RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error { + s.lastSeq = evt.Seq + did, err := syntax.ParseDID(evt.Did) + if err != nil { + s.logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + return nil + } + if err := s.engine.ProcessIdentityEvent(ctx, "handle", did); err != nil { + s.logger.Error("processing handle update failed", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + } + return nil + }, + // TODO: other event callbacks as needed + } + + // start at higher parallelism (somewhat arbitrary) + scaleSettings := autoscaling.DefaultAutoscaleSettings() + scaleSettings.Concurrency = 6 + return events.HandleRepoStream( + ctx, con, autoscaling.NewScheduler( + scaleSettings, + s.bgshost, + rsc.EventHandler, + ), + ) +} + +// NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better. +func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { + + logger := s.logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq) + logger.Debug("received commit event") + + if evt.TooBig { + logger.Warn("skipping tooBig events for now") + return nil + } + + did, err := syntax.ParseDID(evt.Repo) + if err != nil { + logger.Error("bad DID syntax in event", "err", err) + return nil + } + + rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + logger.Error("failed to read repo from car", "err", err) + return nil + } + + for _, op := range evt.Ops { + logger = logger.With("eventKind", op.Action, "path", op.Path) + + ek := repomgr.EventKind(op.Action) + switch ek { + case repomgr.EvtKindCreateRecord: + // read the record from blocks, and verify CID + rc, rec, err := rr.GetRecord(ctx, op.Path) + if err != nil { + logger.Error("reading record from event blocks (CAR)", "err", err) + break + } + if op.Cid == nil || lexutil.LexLink(rc) != *op.Cid { + logger.Error("mismatch between commit op CID and record block", "recordCID", rc, "opCID", op.Cid) + break + } + + err = s.engine.ProcessRecord(ctx, did, op.Path, op.Cid.String(), rec) + if err != nil { + logger.Error("engine failed to process record", "err", err) + continue + } + default: + // TODO: other event types: update, delete + } + } + + return nil +} diff --git a/cmd/hepa/main.go b/cmd/hepa/main.go new file mode 100644 index 000000000..c652c432b --- /dev/null +++ b/cmd/hepa/main.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/automod" + + "github.com/carlmjohnson/versioninfo" + _ "github.com/joho/godotenv/autoload" + cli "github.com/urfave/cli/v2" + "golang.org/x/time/rate" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.App{ + Name: "hepa", + Usage: "automod daemon (cleans the atmosphere)", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "atp-bgs-host", + Usage: "hostname and port of BGS to subscribe to", + Value: "wss://bsky.network", + EnvVars: []string{"ATP_BGS_HOST"}, + }, + &cli.StringFlag{ + Name: "atp-plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + EnvVars: []string{"ATP_PLC_HOST"}, + }, + &cli.StringFlag{ + Name: "atp-mod-host", + Usage: "method, hostname, and port of moderation service", + Value: "https://api.bsky.app", + EnvVars: []string{"ATP_MOD_HOST"}, + }, + &cli.StringFlag{ + Name: "atp-bsky-host", + Usage: "method, hostname, and port of bsky API (appview) service", + Value: "https://api.bsky.app", + EnvVars: []string{"ATP_BSKY_HOST"}, + }, + &cli.StringFlag{ + Name: "redis-url", + Usage: "redis connection URL", + // redis://:@localhost:6379/ + // redis://localhost:6379/0 + EnvVars: []string{"HEPA_REDIS_URL"}, + }, + &cli.StringFlag{ + Name: "mod-handle", + Usage: "for mod service login", + EnvVars: []string{"HEPA_MOD_AUTH_HANDLE"}, + }, + &cli.StringFlag{ + Name: "mod-password", + Usage: "for mod service login", + EnvVars: []string{"HEPA_MOD_AUTH_PASSWORD"}, + }, + &cli.StringFlag{ + Name: "mod-admin-token", + Usage: "admin authentication password for mod service", + EnvVars: []string{"HEPA_MOD_AUTH_ADMIN_TOKEN"}, + }, + &cli.IntFlag{ + Name: "plc-rate-limit", + Usage: "max number of requests per second to PLC registry", + Value: 100, + EnvVars: []string{"HEPA_PLC_RATE_LIMIT"}, + }, + &cli.StringFlag{ + Name: "sets-json-path", + Usage: "file path of JSON file containing static sets", + EnvVars: []string{"HEPA_SETS_JSON_PATH"}, + }, + } + + app.Commands = []*cli.Command{ + runCmd, + processRecordCmd, + } + + return app.Run(args) +} + +func configDirectory(cctx *cli.Context) (identity.Directory, error) { + baseDir := identity.BaseDirectory{ + PLCURL: cctx.String("atp-plc-host"), + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + PLCLimiter: rate.NewLimiter(rate.Limit(cctx.Int("plc-rate-limit")), 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social", ".staging.bsky.dev"}, + } + var dir identity.Directory + if cctx.String("redis-url") != "" { + rdir, err := automod.NewRedisDirectory(&baseDir, cctx.String("redis-url"), time.Hour*24, time.Minute*2) + if err != nil { + return nil, err + } + dir = rdir + } else { + cdir := identity.NewCacheDirectory(&baseDir, 1_500_000, time.Hour*24, time.Minute*2) + dir = &cdir + } + return dir, nil +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "run the hepa daemon", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for metrics APIs", + Value: ":3989", + EnvVars: []string{"HEPA_METRICS_LISTEN"}, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + slog.SetDefault(logger) + + configOTEL("hepa") + + dir, err := configDirectory(cctx) + if err != nil { + return err + } + + srv, err := NewServer( + dir, + Config{ + BGSHost: cctx.String("atp-bgs-host"), + BskyHost: cctx.String("atp-bsky-host"), + Logger: logger, + ModHost: cctx.String("atp-mod-host"), + ModAdminToken: cctx.String("mod-admin-token"), + ModUsername: cctx.String("mod-handle"), + ModPassword: cctx.String("mod-password"), + SetsFileJSON: cctx.String("sets-json-path"), + RedisURL: cctx.String("redis-url"), + }, + ) + if err != nil { + return err + } + + // prometheus HTTP endpoint: /metrics + go func() { + if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil { + slog.Error("failed to start metrics endpoint", "error", err) + panic(fmt.Errorf("failed to start metrics endpoint: %w", err)) + } + }() + + go func() { + if err := srv.RunPersistCursor(ctx); err != nil { + slog.Error("cursor routine failed", "err", err) + } + }() + + // the main service loop + if err := srv.RunConsumer(ctx); err != nil { + return fmt.Errorf("failure consuming and processing firehose: %w", err) + } + return nil + }, +} + +var processRecordCmd = &cli.Command{ + Name: "process-record", + Usage: "process a single record in isolation", + ArgsUsage: ``, + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + uri := cctx.Args().First() + if uri == "" { + return fmt.Errorf("expected a single AT-URI argument") + } + + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + slog.SetDefault(logger) + + dir, err := configDirectory(cctx) + if err != nil { + return err + } + + srv, err := NewServer( + dir, + Config{ + BGSHost: cctx.String("atp-bgs-host"), + BskyHost: cctx.String("atp-bsky-host"), + Logger: logger, + ModHost: cctx.String("atp-mod-host"), + ModAdminToken: cctx.String("mod-admin-token"), + ModUsername: cctx.String("mod-handle"), + ModPassword: cctx.String("mod-password"), + SetsFileJSON: cctx.String("sets-json-path"), + RedisURL: cctx.String("redis-url"), + }, + ) + if err != nil { + return err + } + + return srv.engine.FetchAndProcessRecord(ctx, uri) + }, +} diff --git a/cmd/hepa/otel.go b/cmd/hepa/otel.go new file mode 100644 index 000000000..b551325c2 --- /dev/null +++ b/cmd/hepa/otel.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "log" + "log/slog" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +var tracer = otel.Tracer("hepa") + +// Enable OTLP HTTP exporter +// For relevant environment variables: +// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables +// At a minimum, you need to set +// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +// TODO: this should be in cliutil or something +func configOTEL(serviceName string) { + if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" { + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + log.Fatal("failed to create trace exporter", "error", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(serviceName), + attribute.String("env", os.Getenv("ENVIRONMENT")), // DataDog + attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } +} diff --git a/cmd/hepa/server.go b/cmd/hepa/server.go new file mode 100644 index 000000000..a09e37c7d --- /dev/null +++ b/cmd/hepa/server.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/rules" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" +) + +type Server struct { + bgshost string + logger *slog.Logger + engine *automod.Engine + rdb *redis.Client + lastSeq int64 +} + +type Config struct { + BGSHost string + BskyHost string + ModHost string + ModAdminToken string + ModUsername string + ModPassword string + SetsFileJSON string + RedisURL string + Logger *slog.Logger +} + +func NewServer(dir identity.Directory, config Config) (*Server, error) { + logger := config.Logger + if logger == nil { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + } + + bgsws := config.BGSHost + if !strings.HasPrefix(bgsws, "ws") { + return nil, fmt.Errorf("specified bgs host must include 'ws://' or 'wss://'") + } + + // TODO: this isn't a very robust way to handle a peristent client + var xrpcc *xrpc.Client + if config.ModAdminToken != "" { + xrpcc = &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: config.ModHost, + AdminToken: &config.ModAdminToken, + Auth: &xrpc.AuthInfo{}, + } + + auth, err := comatproto.ServerCreateSession(context.TODO(), xrpcc, &comatproto.ServerCreateSession_Input{ + Identifier: config.ModUsername, + Password: config.ModPassword, + }) + if err != nil { + return nil, err + } + xrpcc.Auth.AccessJwt = auth.AccessJwt + xrpcc.Auth.RefreshJwt = auth.RefreshJwt + xrpcc.Auth.Did = auth.Did + xrpcc.Auth.Handle = auth.Handle + } + + sets := automod.NewMemSetStore() + if config.SetsFileJSON != "" { + if err := sets.LoadFromFileJSON(config.SetsFileJSON); err != nil { + return nil, err + } else { + logger.Info("loaded set config from JSON", "path", config.SetsFileJSON) + } + } + + var counters automod.CountStore + var cache automod.CacheStore + var rdb *redis.Client + if config.RedisURL != "" { + // generic client, for cursor state + opt, err := redis.ParseURL(config.RedisURL) + if err != nil { + return nil, err + } + rdb = redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, err + } + + cnt, err := automod.NewRedisCountStore(config.RedisURL) + if err != nil { + return nil, err + } + counters = cnt + + csh, err := automod.NewRedisCacheStore(config.RedisURL, 30*time.Minute) + if err != nil { + return nil, err + } + cache = csh + } else { + counters = automod.NewMemCountStore() + cache = automod.NewMemCacheStore(5_000, 30*time.Minute) + } + + engine := automod.Engine{ + Logger: logger, + Directory: dir, + Counters: counters, + Sets: sets, + Cache: cache, + Rules: rules.DefaultRules(), + AdminClient: xrpcc, + BskyClient: &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: config.BskyHost, + }, + } + + s := &Server{ + bgshost: config.BGSHost, + logger: logger, + engine: &engine, + rdb: rdb, + } + + return s, nil +} + +func (s *Server) RunMetrics(listen string) error { + http.Handle("/metrics", promhttp.Handler()) + return http.ListenAndServe(listen, nil) +} + +var cursorKey = "hepa/seq" + +func (s *Server) ReadLastCursor(ctx context.Context) (int64, error) { + // if redis isn't configured, just skip + if s.rdb == nil { + s.logger.Info("redis not configured, skipping cursor read") + return 0, nil + } + + val, err := s.rdb.Get(ctx, cursorKey).Int64() + if err == redis.Nil { + s.logger.Info("no pre-existing cursor in redis") + return 0, nil + } + s.logger.Info("successfully found prior subscription cursor seq in redis", "seq", val) + return val, err +} + +func (s *Server) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if s.rdb == nil { + return nil + } + if s.lastSeq <= 0 { + return nil + } + err := s.rdb.Set(ctx, cursorKey, s.lastSeq, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (s *Server) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if s.rdb == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + if s.lastSeq >= 1 { + s.logger.Info("persisting final cursor seq value", "seq", s.lastSeq) + err := s.PersistCursor(ctx) + if err != nil { + s.logger.Error("failed to persist cursor", "err", err, "seq", s.lastSeq) + } + } + return nil + case <-ticker.C: + if s.lastSeq >= 1 { + err := s.PersistCursor(ctx) + if err != nil { + s.logger.Error("failed to persist cursor", "err", err, "seq", s.lastSeq) + } + } + } + } +} diff --git a/go.mod b/go.mod index 9ecb7ff9c..459f9861c 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/carlmjohnson/versioninfo v0.22.5 github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 github.com/flosch/pongo2/v6 v6.0.0 + github.com/go-redis/cache/v9 v9.0.0 github.com/goccy/go-json v0.10.2 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible @@ -46,6 +47,7 @@ require ( github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 + github.com/redis/go-redis/v9 v9.3.0 github.com/rivo/uniseg v0.1.0 github.com/samber/slog-echo v1.2.1 github.com/scylladb/gocqlx/v2 v2.8.1-0.20230309105046-dec046bd85e6 @@ -76,6 +78,14 @@ require ( gorm.io/plugin/opentelemetry v0.1.3 ) +require ( + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/vmihailenco/go-tinylfu v0.2.2 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect +) + require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index b2aaa467c..76a053df1 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,10 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brianvoe/gofakeit/v6 v6.20.2 h1:FLloufuC7NcbHqDzVQ42CG9AKryS1gAGCRt8nQRsW+Y= github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -99,6 +103,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 h1:6SNWi8VxQeCSwmLuTbEvJd7xvPmdS//zvMBWweZLgck= github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -112,6 +118,9 @@ github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -127,11 +136,15 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= +github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -206,6 +219,7 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -239,9 +253,11 @@ github.com/hashicorp/golang-lru/arc/v2 v2.0.6 h1:4NU7uP5vSoK6TbaMj3NtY478TTAWLso github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno= github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icrowley/fake v0.0.0-20221112152111-d7b7e2276db2 h1:qU3v73XG4QAqCPHA4HOpfC1EfUvtLIDvQK4mNQ0LvgI= github.com/icrowley/fake v0.0.0-20221112152111-d7b7e2276db2/go.mod h1:dQ6TM/OGAe+cMws81eTe4Btv1dKxfPZ2CX+YaAFAPN4= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= @@ -367,6 +383,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= @@ -491,6 +509,32 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/opensearch-project/opensearch-go/v2 v2.2.0 h1:6RicCBiqboSVtLMjSiKgVQIsND4I3sxELg9uwWe/TKM= github.com/opensearch-project/opensearch-go/v2 v2.2.0/go.mod h1:R8NTTQMmfSRsmZdfEn2o9ZSuSXn0WTHPYhzgl7LCFLY= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -542,6 +586,9 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= +github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= +github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= +github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -582,6 +629,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -598,6 +646,12 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= +github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= +github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-testmark v0.11.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -618,6 +672,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/yawning/secp256k1-voi v0.0.0-20230815035612-a7264edccf80 h1:+Hti+G65Kc88hK0GFQ6NzzncsOmoqxmlXaxM1+FPPqM= gitlab.com/yawning/secp256k1-voi v0.0.0-20230815035612-a7264edccf80/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= @@ -684,6 +739,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= @@ -721,12 +777,16 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -749,6 +809,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -757,11 +818,17 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= @@ -791,6 +858,7 @@ golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -802,8 +870,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -824,6 +895,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -833,16 +905,22 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -853,6 +931,9 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= @@ -864,6 +945,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -919,9 +1002,13 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= @@ -1026,8 +1113,11 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=