add update

firehose/firehose-core/CHANGELOG.md

@@ -8,8 +8,9 @@ Operators, you should copy/paste content of this content straight to your projec
 
 If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you should copy the content between those 2 version to your own repository, replacing placeholder value `fire{chain}` with your chain's own binary.
 
-## Unreleased
+## v1.3.6
 
+* bump substreams to v1.5.5 with fix in wazero to prevent process freezing on certain substreams
 * bump go-generics to v3.4.0
 
 ## v1.3.5
@@ -110,15 +111,19 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s
 
 * Fixed `tools check merged-blocks` to be able to run without a block range provided.
 
-* Added API Key authentication to `client.NewFirehoseFetchClient` and `client.NewFirehoseClient`.
-
-  > [!NOTE]
-  > If you were using `github.com/streamingfast/firehose-core/firehose/client.NewFirehoseFetchClient` or `github.com/streamingfast/firehose-core/firehose/client.NewFirehoseStreamClient`, this will be a minor breaking change, refer to [upgrade notes](./UPDATE.md#v125) for details if it affects you.
+* Added API Key based authentication to `tools firehose-client` and `tools firehose-single-block-client`, specify the value through environment variable `FIREHOSE_API_KEY` (you can use flag `--api-key-env-var` to change variable's name to something else than `FIREHOSE_API_KEY`).
 
 * Fixed `tools check merged-blocks` examples using block range (range should be specified as `[<start>]?:[<end>]`).
 
 * Added `--substreams-tier2-max-concurrent-requests` to limit the number of concurrent requests to the tier2 Substreams service.
 
+### Library `firehose-core`
+
+* Added API Key authentication to `client.NewFirehoseFetchClient` and `client.NewFirehoseClient`.
+
+  > [!NOTE]
+  > If you were using `github.com/streamingfast/firehose-core/firehose/client.NewFirehoseFetchClient` or `github.com/streamingfast/firehose-core/firehose/client.NewFirehoseStreamClient`, this will be a minor breaking change, refer to [upgrade notes](./UPDATE.md#v125) for details if it affects you.
+
 ## v1.2.4
 
 ### Substreams improvements

firehose/firehose-core/blockpoller/poller.go

@@ -3,8 +3,6 @@ package blockpoller
 import (
 	"context"
 	"fmt"
-	"math"
-
 	"github.com/streamingfast/bstream"
 	"github.com/streamingfast/bstream/forkable"
 	pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1"
@@ -13,6 +11,7 @@ import (
 	"github.com/streamingfast/firehose-core/internal/utils"
 	"github.com/streamingfast/shutter"
 	"go.uber.org/zap"
+	"math"
 )
 
 type block struct {
@@ -39,6 +38,8 @@ type BlockPoller struct {
 	logger *zap.Logger
 
 	optimisticallyPolledBlocks map[uint64]*BlockItem
+
+	fetching bool
 }
 
 func New(
@@ -103,7 +104,21 @@ func (p *BlockPoller) run(resolvedStartBlock bstream.BlockRef, numberOfBlockToFe
 		if hashToFetch != nil {
 			fetchedBlock, err = p.fetchBlockWithHash(blockToFetch, *hashToFetch)
 		} else {
-			fetchedBlock, err = p.fetchBlock(blockToFetch, numberOfBlockToFetch)
+
+			for {
+				requestedBlockItem := p.requestBlock(blockToFetch, numberOfBlockToFetch)
+				fetchedBlockItem := <-requestedBlockItem
+
+				if fetchedBlockItem.skipped {
+					p.logger.Info("block was skipped", zap.Uint64("block_num", fetchedBlockItem.blockNumber))
+					blockToFetch++
+					continue
+				}
+
+				fetchedBlock = fetchedBlockItem.block
+				break
+			}
+
 		}
 
 		if err != nil {
@@ -179,6 +194,7 @@ type BlockItem struct {
 
 func (p *BlockPoller) loadNextBlocks(requestedBlock uint64, numberOfBlockToFetch int) error {
 	p.optimisticallyPolledBlocks = map[uint64]*BlockItem{}
+	p.fetching = true
 
 	nailer := dhammer.NewNailer(10, func(ctx context.Context, blockToFetch uint64) (*BlockItem, error) {
 		var blockItem *BlockItem

firehose/firehose-core/cmd/tools/print/tools_print.go

@@ -250,9 +250,13 @@ func PrintBStreamBlock(b *pbbstream.Block, printTransactions bool, out io.Writer
 	_, err := out.Write(
 		[]byte(
 			fmt.Sprintf(
-				"Block #%d (%s)\n",
+				"Block #%d (%s)\n - Parent: #%d (%s)\n  - LIB: #%d\n  - Time: %s\n",
 				b.Number,
 				b.Id,
+				b.ParentNum,
+				b.ParentId,
+				b.LibNum,
+				b.Timestamp.AsTime(),
 			),
 		),
 	)

firehose/firehose-core/devel/standard_local/bootstrap.sh

@@ -32,8 +32,8 @@ start:
      reader-node-path: "../../../firehose-extract/target/debug/firehose-extract"
      reader-node-arguments: $CHAIN_ID $LAST_HEIGHT
 
-     common-live-blocks-addr: "localhost:10024"
-     reader-node-grpc-listen-addr: "localhost:10024"
+     common-live-blocks-addr: localhost:10024
+     reader-node-grpc-listen-addr: localhost:10024
 
 END
 

firehose/firehose-core/go.mod

@@ -16,21 +16,21 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.15.0
-	github.com/streamingfast/bstream v0.0.2-0.20240228193450-5200ecab8050
-	github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80
+	github.com/streamingfast/bstream v0.0.2-0.20240409115502-d29a2fb46f37
+	github.com/streamingfast/cli v0.0.4-0.20240412191021-5f81842cb71d
 	github.com/streamingfast/dauth v0.0.0-20240222213226-519afc16cf84
 	github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c
 	github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1
 	github.com/streamingfast/dgrpc v0.0.0-20240222213940-b9f324ff4d5c
 	github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4
-	github.com/streamingfast/dmetering v0.0.0-20240403142935-dc8bb3bb32c3
+	github.com/streamingfast/dmetering v0.0.0-20240409120340-b517f0225538
 	github.com/streamingfast/dmetrics v0.0.0-20230919161904-206fa8ebd545
 	github.com/streamingfast/dstore v0.1.1-0.20240325191553-bcce8892a9bb
 	github.com/streamingfast/jsonpb v0.0.0-20210811021341-3670f0aa02d0
 	github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091
 	github.com/streamingfast/pbgo v0.0.6-0.20240131193313-6b88bc7139db
 	github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0
-	github.com/streamingfast/substreams v1.5.3-0.20240405214647-fa91cf14ac98
+	github.com/streamingfast/substreams v1.5.5
 	github.com/stretchr/testify v1.8.4
 	github.com/test-go/testify v1.1.4
 	go.uber.org/multierr v1.10.0
@@ -44,6 +44,7 @@ require (
 	connectrpc.com/grpchealth v1.3.0 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
+	github.com/bobg/go-generics/v3 v3.4.0 // indirect
 	github.com/bufbuild/protocompile v0.4.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
@@ -82,7 +83,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.12.0 // indirect
 	github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect
-	github.com/bobg/go-generics/v2 v2.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -179,7 +179,7 @@ require (
 	github.com/streamingfast/shutter v1.5.0
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect
-	github.com/tetratelabs/wazero v1.7.0 // indirect
+	github.com/tetratelabs/wazero v1.7.1 // indirect
 	github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect
 	github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 // indirect
 	go.opencensus.io v0.24.0

firehose/firehose-core/go.sum

@@ -148,8 +148,8 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6
 github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc=
 github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d h1:fSlGu5ePbkjBidXuj2O5j9EcYrVB5Cr6/wdkYyDgxZk=
 github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d/go.mod h1:yCBkgASmKHgUOFjK9h1sOytUVgA+JkQjqj3xYP4AdWY=
-github.com/bobg/go-generics/v2 v2.1.1 h1:4rN9upY6Xm4TASSMeH+NzUghgO4h/SbNrQphIjRd/R0=
-github.com/bobg/go-generics/v2 v2.1.1/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk=
+github.com/bobg/go-generics/v3 v3.4.0 h1:XxTJxH843OknMgw//HGQXklJCZ0eacdt5EABfNcKFr8=
+github.com/bobg/go-generics/v3 v3.4.0/go.mod h1:gCsHnnRz88zpXpdsWPyDmjg1tYQPmpbUQbM4MW8z9Jc=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
@@ -584,10 +584,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/streamingfast/bstream v0.0.2-0.20240228193450-5200ecab8050 h1:S+1qRKKco3h+7q1voMHPDvX7gewJebs0ZG7aS0XKvZk=
-github.com/streamingfast/bstream v0.0.2-0.20240228193450-5200ecab8050/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y=
-github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 h1:UxJUTcEVkdZy8N77E3exz0iNlgQuxl4m220GPvzdZ2s=
-github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80/go.mod h1:QxjVH73Lkqk+mP8bndvhMuQDUINfkgsYhdCH/5TJFKI=
+github.com/streamingfast/bstream v0.0.2-0.20240409115502-d29a2fb46f37 h1:nh6BwIY51Anirm1G/w8zZ3XAnOLF1CF2GUzetBh9J7o=
+github.com/streamingfast/bstream v0.0.2-0.20240409115502-d29a2fb46f37/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y=
+github.com/streamingfast/cli v0.0.4-0.20240412191021-5f81842cb71d h1:9tsEt2tLCp94CW6MyJZY+Rw6+t0WH2kioBR6ucO6P/E=
+github.com/streamingfast/cli v0.0.4-0.20240412191021-5f81842cb71d/go.mod h1:og+6lDBPLZ24lbF/YISmVsSduZUZwXSmJGD3pZ/sW2Y=
 github.com/streamingfast/dauth v0.0.0-20240222213226-519afc16cf84 h1:yCvuNcwQ21J4Ua6YrAmHDBx3bjK04y+ssEYBe65BXRU=
 github.com/streamingfast/dauth v0.0.0-20240222213226-519afc16cf84/go.mod h1:cwfI5vaMd+CiwZIL0H0JdP5UDWCZOVFz/ex3L0+o/j4=
 github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c h1:6WjE2yInE+5jnI7cmCcxOiGZiEs2FQm9Zsg2a9Ivp0Q=
@@ -598,8 +598,8 @@ github.com/streamingfast/dgrpc v0.0.0-20240222213940-b9f324ff4d5c h1:hn5ZPKGtgsc
 github.com/streamingfast/dgrpc v0.0.0-20240222213940-b9f324ff4d5c/go.mod h1:EPtUX/vhRphE37Zo6sDcgD/S3sm5YqXHhxAgzS6Ebwo=
 github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4 h1:HKi8AIkLBzxZWmbCRUo1RxoOLK33iXO6gZprfsE9rf4=
 github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4/go.mod h1:ehPytv7E4rI65iLcrwTes4rNGGqPPiugnH+20nDQyp4=
-github.com/streamingfast/dmetering v0.0.0-20240403142935-dc8bb3bb32c3 h1:u3C2jzTc7d58PvVjlZew4HmZ1g1xr9yWBd8eWjmQNig=
-github.com/streamingfast/dmetering v0.0.0-20240403142935-dc8bb3bb32c3/go.mod h1:UqWuX3REU/IInBUaymFN2eLjuvz+/0SsoUFjeQlLNyI=
+github.com/streamingfast/dmetering v0.0.0-20240409120340-b517f0225538 h1:9zCunCSc2ZeNrbOsz6v31Qo6Dfr7AYADfhhmr1V3St8=
+github.com/streamingfast/dmetering v0.0.0-20240409120340-b517f0225538/go.mod h1:UqWuX3REU/IInBUaymFN2eLjuvz+/0SsoUFjeQlLNyI=
 github.com/streamingfast/dmetrics v0.0.0-20230919161904-206fa8ebd545 h1:SUl04bZKGAv207lp7/6CHOJIRpjUKunwItrno3K463Y=
 github.com/streamingfast/dmetrics v0.0.0-20230919161904-206fa8ebd545/go.mod h1:JbxEDbzWRG1dHdNIPrYfuPllEkktZMgm40AwVIBENcw=
 github.com/streamingfast/dstore v0.1.1-0.20240325191553-bcce8892a9bb h1:tmu8wGiSTzdqk2CnPnI7GywKwepGieqNOQDRKKSiVJg=
@@ -648,8 +648,8 @@ github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ7
 github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
 github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
 github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
-github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ=
-github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
+github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
+github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
 github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk=
 github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0=
 github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=

firehose/firehose-core/stream_factory.go

@@ -89,6 +89,9 @@ func (sf *StreamFactory) New(
 			zap.String("user_id", auth.UserID()),
 			zap.String("real_ip", auth.RealIP()),
 		)
+		if auth["x-deployment-id"] != "" {
+			fields = append(fields, zap.String("deployment_id", auth["x-deployment-id"]))
+		}
 	}
 
 	reqLogger.Info("processing incoming blocks request", fields...)

firehose/substreams/docs/new/common/packages.md

@@ -2,22 +2,23 @@
 description: Learn about basics of Substreams packages
 ---
 
-There are a lot of developers building Substreams and creating very useful transformations that can be reused by other people.
-Once a Substreams is developed, you can pack it into a Substreams package and share it with other people!
+# Packages
+
+There are a lot of developers building Substreams and creating very useful transformations that can be reused by other people. Once a Substreams is developed, you can pack it into a Substreams package and share it with other people!
 
 Essentially, a Substreams package is a ready-to-consume binary file, which contains all the necessary dependencies (manifest, modules, protobufs...). The standard file extension for a Substreams package is `.spkg`.
 
-<figure><img src="../../.gitbook/assets/packages/arch.png" width="100%" /></figure>
+<figure><img src="../../.gitbook/assets/packages/arch.png" alt="" width="100%"><figcaption></figcaption></figure>
 
-## The Substreams Registry
+### The Substreams Registry
 
 In order to facilitate how developers share Substreams packages, the Substreams Registry (https://substreams.dev) was created. In the Registry, developers can discover and push Substreams.
 
 For example, the [ERC20 Balance Changes](https://github.com/streamingfast/substreams-erc20-balance-changes) package is stored at the registry (https://substreams.dev/streamingfast/erc20-balance-changes/v1.1.0).
 
-<figure><img src="../../.gitbook/assets/packages/erc20-balance-changes.png" width="100%" /></figure>
+<figure><img src="../../.gitbook/assets/packages/erc20-balance-changes.png" alt="" width="100%"><figcaption></figcaption></figure>
 
-## Using a Package
+### Using a Package
 
 You can easily run a Substreams package by inputting the `.spkg` file in the CLI:
 
@@ -29,20 +30,20 @@ substreams gui \
  --start-block 1397553
 ```
 
-## Creating a Package
+### Creating a Package
 
 You can create a Substreams package by executing the `substreams pack` command in the CLI. Given a Substreams project, you can create a new package from a manifest (`substreams.yaml`):
 
 ```bash
 substreams pack ./substreams.yaml
 ```
 
-### Package Dependencies
+#### Package Dependencies
 
 Developers can use modules and protobuf definitions from other Substreams packages when `imports` is defined in the manifest.
 
 {% hint style="warning" %}
 **Important**: To avoid potential naming collisions select unique `.proto` filenames and namespaces specifying fully qualified paths.
 {% endhint %}
 
-Local protobuf filenames take precedence over the imported package's proto files.
+Local Protobuf filenames take precedence over the imported package's proto files.

firehose/substreams/docs/release-notes/change-log.md

@@ -9,6 +9,29 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v1.5.5 (unreleased)
+
+### Fixes
+
+* bump wazero execution to fix issue with certain substreams causing the server process to freeze
+
+### Changes
+
+* Allow unordered ordinals to be applied from the substreams (automatic ordering before flushing to stores)
+
+### Add
+
+* add `substreams_tier1_worker_retry_counter` metric to count all worker errors returned by tier2
+* add `substreams_tier1_worker_rejected_overloaded_counter` metric to count only worker errors with string "service currently overloaded"
+
+## v1.5.4
+
+### Fixes
+
+* fix a possible panic() when an request is interrupted during the file loading phase of a squashing operation.
+* fix a rare possibility of stalling if only some fullkv stores caches were deleted, but further segments were still present.
+* fix stats counters for store operations time
+
 ## v1.5.3
 
 Performance, memory leak and bug fixes

firehose/substreams/go.mod

@@ -60,7 +60,7 @@ require (
 	github.com/streamingfast/shutter v1.5.0
 	github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231127153906-acf5f3e34330
 	github.com/test-go/testify v1.1.4
-	github.com/tetratelabs/wazero v1.7.0
+	github.com/tetratelabs/wazero v1.7.1
 	github.com/tidwall/pretty v1.2.1
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0
 	go.opentelemetry.io/otel v1.24.0

firehose/substreams/go.sum

@@ -629,8 +629,8 @@ github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ7
 github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
 github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
 github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
-github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ=
-github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
+github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
+github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
 github.com/thedevsaddam/govalidator v1.9.6 h1:hWLpGOHkDjzM+uQh4xankDUm8dgXaRft/B1/sDAShKU=
 github.com/thedevsaddam/govalidator v1.9.6/go.mod h1:Ilx8u7cg5g3LXbSS943cx5kczyNuUn7LH/cK5MYuE90=
 github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=

firehose/substreams/metrics/metrics.go

@@ -21,6 +21,8 @@ var SquashersEnded = MetricSet.NewCounter("substreams_total_squash_processes_clo
 
 var Tier1ActiveWorkerRequest = MetricSet.NewGauge("substreams_tier1_active_worker_requests", "Number of active Substreams worker requests a tier1 app is currently doing against tier2 nodes")
 var Tier1WorkerRequestCounter = MetricSet.NewCounter("substreams_tier1_worker_request_counter", "Counter for total Substreams worker requests a tier1 app made against tier2 nodes")
+var Tier1WorkerRetryCounter = MetricSet.NewCounter("substreams_tier1_worker_retry_counter", "Counter for total retryable errors returned from tier2")
+var Tier1WorkerRejectedOverloadedCounter = MetricSet.NewCounter("substreams_tier1_worker_rejected_overloaded_counter", "Counter for number of times a worker rejected a request because it was overloaded (included in RetryCounter)")
 
 var Tier2ActiveRequests = MetricSet.NewGauge("substreams_tier2_active_requests", "Number of active Substreams requests the tier2 is currently serving")
 var Tier2RequestCounter = MetricSet.NewCounter("substreams_tier2_request_counter", "Counter for total Substreams requests the tier2 served")