diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000..2f8ed0acd --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,43 @@ +name: build-test + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "develop" + +jobs: + audit: + runs-on: ubuntu-20.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.22.5 + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Verify dependencies + run: go mod verify + + - name: Build all + run: make all + + - name: Run tests + run: make test + + - name: Upload coverage to Codecov + run: bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }} + + - name: Clean + run: make clean diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 000000000..c0f5ed78e --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,46 @@ +name: docker-image-ci + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "develop" + +jobs: + push-image: + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.CR_USER }} + password: ${{ secrets.CR_PAT }} + + - uses: actions/checkout@v2 + - name: Build & Publish the Docker image + if: ${{ !env.ACT }} + run: | + docker build . --file Dockerfile --tag ghcr.io/covalenthq/bsp-geth:latest + docker push ghcr.io/covalenthq/bsp-geth:latest + + # - name: Start containers + # run: docker-compose -f "docker-compose.yml" up --build --remove-orphans --force-recreate --exit-code-from agent + + # - name: Check running bsp-geth + # run: docker inspect bsp-geth + + # - name: Check running containers + # run: docker ps + + # - name: Stop containers + # if: always() + # run: docker-compose -f "docker-compose.yml" down diff --git a/.github/workflows/gcr-image.yml b/.github/workflows/gcr-image.yml new file mode 100644 index 000000000..917f35175 --- /dev/null +++ b/.github/workflows/gcr-image.yml @@ -0,0 +1,31 @@ +name: gcr-image + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "develop" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GCR + uses: docker/login-action@v2 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.GCR_JSON_KEY }} + + - uses: actions/checkout@v2 + # - uses: satackey/action-docker-layer-caching@v0.0.11 + - name: Build & Publish the Docker image + run: | + docker buildx create --name builder --use --platform=linux/amd64,linux/arm64 && docker buildx build --platform=linux/amd64,linux/arm64 . -t us-docker.pkg.dev/covalent-project/network/bsp-geth:latest --push diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6406d2fa3..be510101b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,21 +2,20 @@ name: i386 linux tests on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] workflow_dispatch: jobs: build: runs-on: self-hosted steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v2 with: - go-version: 1.23.0 - cache: false + go-version: 1.22.5 - name: Run tests run: go test -short ./... env: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 000000000..caaf41930 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,50 @@ +name: golangci-lint + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "develop" +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read +jobs: + golangci-build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + - name: Set up Go 1.22.5 + uses: actions/setup-go@v3 + with: + go-version: 1.22.5 + id: go + - run: go version + + - name: Lint + run: | + # curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.51.2 + # ./bin/golangci-lint run + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # args: --issues-exit-code=0 + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the action will use pre-installed Go. + # skip-go-installation: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml new file mode 100644 index 000000000..a120b7792 --- /dev/null +++ b/.github/workflows/hadolint.yml @@ -0,0 +1,21 @@ +name: dockerfile-lint + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "develop" + +jobs: + linter: + name: lint-dockerfile + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: lint dockerfile + uses: brpaz/hadolint-action@master + with: + dockerfile: "Dockerfile" diff --git a/.github/workflows/tag-release.yaml b/.github/workflows/tag-release.yaml new file mode 100644 index 000000000..c8b9028a8 --- /dev/null +++ b/.github/workflows/tag-release.yaml @@ -0,0 +1,39 @@ +name: tag-release + +on: + push: + tags: + - "v*.*.*" + +jobs: + tagged-release: + name: Tagged Release + runs-on: "ubuntu-latest" + + steps: + - name: Login to GCR + uses: docker/login-action@v2 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.GCR_JSON_KEY }} + + - uses: actions/checkout@v2 + + # - uses: satackey/action-docker-layer-caching@v0.0.11 + + - name: Set env + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Build & Publish the Docker image + run: | + docker buildx create --name builder --use --platform=linux/amd64,linux/arm64 && docker buildx build --platform=linux/amd64,linux/arm64 . -t us-docker.pkg.dev/covalent-project/network/bsp-geth:stable -t us-docker.pkg.dev/covalent-project/network/bsp-geth:"${{ env.TAG }}" --push + + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + draft: false + prerelease: false + files: | + *.zip + *.tar.gz diff --git a/.gitignore b/.gitignore index b62ff4c00..067bba38a 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,23 @@ profile.cov # VS Code .vscode +# dashboard +/dashboard/assets/flow-typed +/dashboard/assets/node_modules +/dashboard/assets/stats.json +/dashboard/assets/bundle.js +/dashboard/assets/bundle.js.map +/dashboard/assets/package-lock.json + +**/yarn-error.log + +data/ +ganache_data/ +.env +scripts/ +logs/ +coverage.out +coverage.txt tests/spec-tests/ -bin/ \ No newline at end of file +bin/ +out/ diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..3cc66d39f --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,4 @@ +ignored: + - DL3018 + - DL3016 + - DL3059 diff --git a/Dockerfile b/Dockerfile index ff89e92f2..005099128 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.23-alpine AS builder +FROM golang:1.22-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git @@ -23,7 +23,7 @@ RUN apk add --no-cache ca-certificates COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp -ENTRYPOINT ["geth"] +ENTRYPOINT ["geth", "--mainnet", "--syncmode", "full", "--datadir", "/root/.ethereum/covalent", "--replication.targets", "redis://localhost:6379/?topic=replication", "--replica.result", "true", "--replica.specimen", "true", "--replica.blob", "true"] # Add some metadata labels to help programmatic image consumption ARG COMMIT="" diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 44d5065af..79cd26b43 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.23-alpine AS builder +FROM golang:1.22-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/README.md b/README.md index 182464c63..bbd66658b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,366 @@ -## Go Ethereum +![banner](./docs/covalent.jpg) + +
+ + Version + + + License: + + + Go report card + + + Discord + + + Lines of code + + + + +
+
+ + Discord + + + Lint Status + + + CI Status + + + Test Status + + + Code Coverage + + + Twitter Follow Covalent + +
+ +# Block Specimen Producer (BSP Geth) + +* [Introduction](#bsp_intro) + * [Resources](#bsp_resources) +* [Raison d'être](#bsp_why) +* [Architecture](#bsp_arch) +* [Docker Run](#docker) +* [Build & Run](#build_run) + * [Flag Definitions](#flag_definitions) +* [Contributing](./docs/CONTRIBUTING.md) +* [Go Ethereum](#geth) + +## Introduction + +Essential to the Covalent Network is the Block Specimen and the Block Specimen Producer (BSP), a bulk export method that ultimately leads to the generation of a canonical representation of a blockchain's historical state. Currently implemented on existing blockchain clients running Geth. It functions currently as an - + +1. Blockchain data extractor +1. Blockchain data normalizer + +What is ultimately created is a ‘Block Specimen’, a universal canonical representation of a blockchain's historical state. + +There are two further considerations regarding the Block Specimen. + +1. The BSP is completely standalone on forks of Geth. +1. The separation of data storage layer from the block execution and distributed consensus functionality leads to better segregation and upgrades of functionality in the blockchain data processing pipeline. + +As a result, anyone can run full tracing on the block specimen and accurately recreate the blockchain without access to a blockchain client software. + +## Resources + +Production of Block Specimens forms the core of the network’s data objects specification. These objects are created with the aid of three main pieces of open-source software provided by Covalent for the network’s decentralized stack. + +1. [Block Specimen Producer (BSP Geth)](https://github.com/covalenthq/bsp-geth) - Operator run & deployed + +1. [BSP Agent](https://github.com/covalenthq/bsp-agent) - Operator run & deployed + +1. [BSP Proof-chain](https://github.com/covalenthq/bsp-staking) - Covalent operated & pre-deployed + +Please refer to these [instructions](https://docs.google.com/document/d/1N_HxUi6ZEkub9EHANe49vkL9iQztVA_ACyfHcOZV5y0/edit?usp=sharing) for running the BSP with the bsp-agent (BSP Agent). + +Please refer to this [whitepaper](https://www.covalenthq.com/static/documents/Block%20Specimen%20Whitepaper%20V1.1.pdf) to understand more about its function. + + +## Raison d'être + + +The blockchain space has been and will continue to be laser-focused on *write* scalability. That is, actually writing on the blockchain (confirming a transaction) and doing so efficiently. And the projects tackling this issue, be it Layer 1s or Layer 2s, have certainly made operating in this space more accommodating, leading to increased adoption as powerful and scalable applications are being developed. + +However, this is only one side of the scalability issue that troubles the space. On the flip side you have the issue of *read scalability*. This is different to *write* as the focus with *read* is on extracting and reading the data on the blockchain, whether that be Ethereum, Avalanche or Solana. + +One common method of reading data from Ethereum for example is the JSON RPC Layer. A number of issues present themselves however when doing so. + +- **Slow**: One needs to make a series of individual data queries to extract the block and its constituent elements like transactions and receipts. + +- **Not Multiversion**: Multiversion concurrency control methods are traditionally employed in databases to ensure point-in-time consistent views if multiple parties are viewing or querying the database. Such methods do not exist in web3. + +- **Expensive**: To access historical data at any point in time, you need to run your blockchain clients in a mode known as “full archive nodes” - which requires specialized and expensive hardware to scale. + +- **The Purge:** For Ethereum specifically, Vitalik recently outlined an updated roadmap for its development which included a phase titled ‘The Purge’. Once this phase is implemented, clients will no longer store historical data older than a year. Hence, alternatives will be needed to access Ethereum's full historical state. + +Meanwhile, data mappers and static dashboards are great for examining specific metrics and small tables (so long as the smart contract is decoded) but lack flexibility and scalability. Our belief is that - + +1. fast, cheap and accessible read capabilities will lead to more diverse and better-adapted blockchain technologies. + +1. should be accessible to all, no matter the skill level. + +The Block Specimen is **the solution** to tackle the read scalability issues that currently plague blockchains. + +## Architecture + +![diagram](./docs/arch.jpg) + +While Block Specimens are currently being created internally at Covalent for each respective blockchain indexed, the Covalent Network shifts this responsibility to operators (anyone performing a role on the Covalent Network). Any operator on the network will be able to opt in to act as a Block Specimen Producer (BSP). + +To ensure that the data within the block-specimens that operators create is reliable and honest, a production proof is created for every Block Specimen produced. These will be published to proofing contract deployed by Covalent. Therefore, Block Specimen proofs can be compared and any deviations in the data either accidentally or malicious will have mismatching proofs. + + +![diagram](./docs/value.png) + +In sum, it is the responsibility of the BSPs to consume blocks from external blockchains and publish both the BSP along with a production proof to the Covalent virtual chain. As the network is developed, Covalent will be shifting the responsibility of running the Block Specimen software to operators (anyone performing a role on the Covalent Network). These Block Specimens will consume blocks from external blockchains and feed the entire network with the data needed to answer user queries. + +How does this create and accrue value for the Covalent Network? As Block Specimen Producers publish more data to Covalent Network, developers will be attracted. With a growing developer base on the Covalent Network, there will be a greater appetite for blockchain data. And hence, the cycle begins again, with Block Specimens expected to meet this demand for data. +Of course, operators who successfully perform this role will be compensated in CQT. + +## Docker Run + +Please install [docker and docker-compose](https://docs.docker.com/compose/install/). + +Employ `docker-compose` to get all the necessary services along with BSP Geth to also get running alongside it. With `docker-compose` BSP Geth creates block-specimens that are extracted Live for Ethereum Mainnet, pushes them to the Redis stream queue, the BSP Agent service then reads the RLP block-specimens and processes the messages from the redis stream and stores/uploads them according to the configuration given to it. The BSP Agent makes statements about the block-specimens by writing to a proxy proof-chain smart contract that is deployed on a test ganache blockchain i.e the proving aspects happen locally and not on a public blockchain. The Specimens stored locally or uploaded to google cloud storage however retain full validity (if the BSP Geth / Agent source code is not altered.) Add an .env file (if needed) to accommodate the env vars. + +In the future, for the BSP Agent the flags `--eth-client` pointing to a Mainnet ethereum client & `--proof-chain-address` pointing to the correct version of the deployed proofing contract along with the an `.env` file with `ETH_PRIVATE_KEY` var will have to be updated to run entirely for CQT Mainnet. +These changes can also be directly adapted into the docker-compose.yml file in this directory. + +The list of all services are - + +1. redis-srv (Open source (BSD licensed), in-memory data structure store) +1. redis-commander-web (Redis web management tool written in node.js) +1. ganache-cli (Ethereum blockchain & client) +1. proof-chain (Validation (proofing) smart-contracts) +1. bsp-agent (Block specimen decoder, packer, prover and storer) + +```bash +git clone git@github.com:covalenthq/bsp-geth.git +cd bsp-geth +docker-compose -f "docker-compose-testnet.yml" up +``` + +If all the services are up and running well, expect to see the logs similar to the following, in approx ~ 10 mins, as the node begins to sync and export Block Specimens. Please note we don't advise running bsp-geth and bsp-agent in production with docker (this is only for demo purposes). + +```bash +bsp-geth | INFO [02-04|18:59:33.731|core/block_replica.go:36] Creating block replication event block number=139 hash=0x41d2931a4495deabbf9f58181a48d29c89036c8fb8b9ecedb5f23805cc6f5e34 +bsp-geth | INFO [02-04|18:59:33.745|core/block_replica.go:36] Creating block replication event block number=140 hash=0xe2c1e8200ef2e9fba09979f0b504dc52c068719623c7064904c7bd3e9365acc1 +ganache-cli | +ganache-cli | Transaction: 0x398c2d9c820a6bbdfd7de696b7e049b25114285f00a4c12f49ef492bf2522858 +ganache-cli | Gas usage: 48457 +ganache-cli | Block Number: 9 +ganache-cli | Block Time: Fri Feb 04 2022 18:59:33 GMT+0000 (Coordinated Universal Time) +ganache-cli | +ganache-cli | eth_getTransactionReceipt +bsp-geth | INFO [02-04|18:59:33.819|core/block_replica.go:36] Creating block replication event block number=141 hash=0xeafbe76fdcadc1b69ba248589eb2a674b60b00c84374c149c9deaf5596183932 +bsp-agent | time="2022-02-04T18:59:33Z" level=info msg="Proof-chain tx hash: 0x398c2d9c820a6bbdfd7de696b7e049b25114285f00a4c12f49ef492bf2522858 for block-replica segment: 1-1-10-replica-segment" function=EncodeProveAndUploadReplicaSegment line=63 +bsp-agent | time="2022-02-04T18:59:33Z" level=info msg="File written successfully to: ./bin/block-ethereum/1-1-10-replica-segment-0x398c2d9c820a6bbdfd7de696b7e049b25114285f00a4c12f49ef492bf2522858" function=writeToBinFile line=88 +bsp-geth | INFO [02-04|18:59:33.855|core/block_replica.go:36] Creating block replication event block number=142 hash=0x8ff76dc49f9a1492813a281a474f102890cdd5a42399241d5fa403f201a4d7cf +bsp-agent | +bsp-agent | ---> Processing 1-11-20-replica-segment <--- +bsp-agent | time="2022-02-04T18:59:33Z" level=info msg="Submitting block-replica segment proof for: 1-11-20-replica-seg +``` + +with occasional responses from `bsp-agent` service such as - + +```bash +bsp-agent | ---> Processing 1-61-70-replica-segment <--- +bsp-agent | time="2022-02-04T19:00:04Z" level=info msg="Submitting block-replica segment proof for: 1-61-70-replica-segment" function=EncodeProveAndUploadReplicaSegment line=57 +bsp-agent | time="2022-02-04T19:00:04Z" level=info msg="Proof-chain tx hash: 0x85b5e7cfa946f3b44b811dce48715841f40627dffb11ebcecb77e3e4a8ef3711 for block-replica segment: 1-61-70-replica-segment" function=EncodeProveAndUploadReplicaSegment line=63 +bsp-agent | time="2022-02-04T19:00:04Z" level=info msg="File written successfully to: ./bin/block-ethereum/1-61-70-replica-segment-0x85b5e7cfa946f3b44b811dce48715841f40627dffb11ebcecb77e3e4a8ef3711" function=writeToBinFile line=88 +``` + +To inspect the actual block specimen binary files produced, exec into the `bsp-agent` container and run the `bsp-extractor` as shown below - + +![diagram](./docs/extract.png) + +```bash +docker ps +docker exec -it /bin/bash +./bsp-extractor --binary-file-path "./bin/block-ethereum/" --codec-path "./codec/block-ethereum.avsc" --indent-json 0 +``` + +The docker image for this service can be found [here](https://github.com/covalenthq/bsp-geth/pkgs/container/go-ethereum-bsp) + +Run only go-ethereum-bsp with the following, though this will not work if the other services in the docker-compose.yml file aren't also initialized. + +```bash +docker pull ghcr.io/covalenthq/bsp-geth-bsp:latest +docker run ghcr.io/covalenthq/bsp-geth-bsp:latest +``` + +## Build & Run + +Clone the `covalenthq/bsp-geth` repo and checkout the branch that contains the block specimen patch aka `covalent` + +```sh +git clone git@github.com:covalenthq/bsp-geth.git +cd bsp-geth +git checkout main +``` + +Build `geth` from source (install [`Go`](https://go.dev/doc/install) if you don’t have it) and other geth developer tools from root. Make sure you also have [`make`](https://www.gnu.org/software/make/) that is used too build and install bsp-geth. If you need all the go-ethereum development related tools do a `make all`. + +```bash +make geth +``` + +Start redis (our streaming service) with the following. + +```bash +$ redis-server +[28550] 01 Aug 19:29:28 # Warning: no config file specified, using the default config. In order to specify a config file use 'redis-server /path/to/redis.conf' +[28550] 01 Aug 19:29:28 * Server started, Redis version 2.2.12 +[28550] 01 Aug 19:29:28 * The server is now ready to accept connections on port 6379 +``` + +Start redis-cli in a separate terminal so you can see the encoded bsps as they are fed into redis streams. + +```bash +$ redis-cli +127.0.0.1:6379> ping +PONG +``` + +We are now ready to start accepting stream message into redis locally. Kindly note that [redis streams](https://redis.io/docs/manual/data-types/streams/) uses in-memory data structures to store new messages (but those are written to disk at the time of exit / closing the server) and can lead to large memory requirements if not managed properly. + +Now start `geth` from root with the given configuration, here we specify the replication targets (block specimen targets) with redis stream topic key `replication-1`, running `geth` in `full` or `snap` syncmode, exposing the http port for the geth apis are optional. bsp-geth can be run as usually run when run as a full / snap node with the extra flags provided here. + +Prior to executing, please replace `` with correct local username within the `--datadir` flag. Everything else remains the same as given below. + +```bash +./build/bin/geth \ + --mainnet \ + --log.debug \ + --syncmode snap \ + --datadir /scratch/node/ethereum/ \ + --replication.targets "redis://localhost:6379/?topic=replication-1" \ + --replica.result \ + --replica.specimen \ + --log.file "./logs/geth.log" +``` + +`bsp-geth` only produces block specimens for live blocks once state sync is complete. In order to check the status of your sync progress. Connect to the node’s IPC instance to check how far the node is synced. + +```bash +./build/bin/geth attach /scratch/node/ethereum/geth.ipc +``` + +Once connected wait for the node at a given `currentBlock` to reach the `highestblock` to start creating live block specimens. `startingBlock` can be 0 or any number depending on the chaindata sync status in `datadir`. + +```bash +Welcome to the Geth JavaScript console! + +instance: Geth/v1.10.17-stable-d1a92cb2/darwin-arm64/go1.17.2 +at block: 10487792 (Mon Apr 11 2022 14:01:59 GMT-0700 (PDT)) + datadir: /scratch/node/ethereum/ + modules: admin:1.0 clique:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 txpool:1.0 web3:1.0 + +To exit, press ctrl-d or type exit +> eth.syncing +{ + currentBlock: 10487906, + healedBytecodeBytes: 0, + healedBytecodes: 0, + healedTrienodeBytes: 0, + healedTrienodes: 0, + healingBytecode: 0, + healingTrienodes: 0, + highestBlock: 10499433, + startingBlock: 10486736, + syncedAccountBytes: 0, + syncedAccounts: 0, + syncedBytecodeBytes: 0, + syncedBytecodes: 0, + syncedStorage: 0, + syncedStorageBytes: 0 +} + +> eth.syncing +false +``` + +This can take a few days or a few hours depending on if the source chaindata is already available at the `datadir` location or live sync is being attempted from scratch for a new copy of blockchain data obtained from syncing with peers. In the case of the latter the strength of the network and other factors that affect the Ethereum network devp2p protocol performance can further cause delays. + +Once blockchain data state sync is complete and `eth.syncing` returns false. You can expect to see block-specimens in the redis stream. The following logs are captured from `bsp-geth` service as the node begins to export live Block Specimens. + +```bash +INFO [04-11|16:35:48.554|core/chain_replication.go:317] Replication progress sessID=1 queued=1 sent=10960 last=0xffc46213ccd3c55b75f73a0bc29c25780eb37f04c9f2b88179e9d0fb889a4151 +INFO [04-11|16:36:04.183|core/blockchain_insert.go:75] Imported new chain segment blocks=1 txs=63 mgas=13.147 elapsed=252.747ms mgasps=52.015 number=10,486,732 hash=8b57c8..bd5c79 dirty=255.49MiB +INFO [04-11|16:36:04.189|core/block_replica.go:41] Creating Block Specimen Exported block=10,486,732 hash=0x8b57c8606d74972c59c56f7fe672a30ed6546fc8169b6a2504abb633aebd5c79 +INFO [04-11|16:36:04.189|core/rawdb/chain_iterator.go:338] Unindexed transactions blocks=1 txs=9 tail=8,136,733 elapsed="369.12µs" +``` + +The last two lines above show that new block replicas containing the block specimens are being produced and streamed to the redis topic “replication”. After this you can check that redis is stacking up the bsp messages through the redis-cli with the command below (this should give you the number of messages from the stream) + +```bash +$ redis-cli +127.0.0.1:6379> xlen replication-1 +11696 +``` + +If it doesn’t - the BSP - producer isn't producing messages! In this case please look at the logs above and see if you have any WARN / DEBUG logs that can be responsible for the inoperation. For quick development iteration and faster network sync - enable a new node key to quickly re-sync with the ethereum network for development and testing by going to the root of go-ethereum and running the bootnode helper. + +NOTE: To use the bootnode binary execute `make all` in place of `make geth`, this creates all the additional helper binaries that `bsp-geth` ships with. + +```bash +./build/bin/bootnode -genkey ~/.ethereum/bsp/geth/nodekey +``` + +Further, also have [`bsp-agent`](https://github.com/covalenthq/bsp-agent) running alongside consuming messages from redis (this will consume the messages and remove them from the stream key). You should see the occasional responses from `bsp-agent` service such as - + +```bash +time="2022-04-18T17:26:47Z" level=info msg="Initializing Consumer: fb78bb1c-1e14-4905-bb1f-0ea96de8d8b5 | Redis Stream: replication-1 | Consumer Group: replicate-1" function=main line=167 +time="2022-04-18T17:26:47Z" level=info msg="block-specimen not created for: 10430548, base block number divisor is :3" function=processStream line=332 +time="2022-04-18T17:26:47Z" level=info msg="stream ids acked and trimmed: [1648848491276-0], for stream key: replication-1, with current length: 11700" function=processStream line=339 +time="2022-04-18T17:26:47Z" level=info msg="block-specimen not created for: 10430549, base block number divisor is :3" function=processStream line=332 +time="2022-04-18T17:26:47Z" level=info msg="stream ids acked and trimmed: [1648848505274-0], for stream key: replication-1, with current length: 11699" function=processStream line=339 + +---> Processing 4-10430550-replica <--- +time="2022-04-18T17:26:47Z" level=info msg="Submitting block-replica segment proof for: 4-10430550-replica" function=EncodeProveAndUploadReplicaSegment line=59 +time="2022-04-18T17:26:47Z" level=info msg="binary file should be available: ipfs://QmUQ4XYJv9syrokUfUbhvA4bV8ce7w1Q2dF6NoNDfSDqxc" function=EncodeProveAndUploadReplicaSegment line=80 +time="2022-04-18T17:27:04Z" level=info msg="Proof-chain tx hash: 0xcc8c487a5db0fec423de62f7ac4ca81c630544aa67c432131cabfa35d9703f37 for block-replica segment: 4-10430550-replica" function=EncodeProveAndUploadReplicaSegment line=86 +time="2022-04-18T17:27:04Z" level=info msg="File written successfully to: /scratch/node/block-ethereum/4-10430550-replica-0xcc8c487a5db0fec423de62f7ac4ca81c630544aa67c432131cabfa35d9703f37" function=writeToBinFile line=188 +time="2022-04-18T17:27:04Z" level=info msg="car file location: /tmp/28077399.car\n" function=generateCarFile line=133 +time="2022-04-18T17:27:08Z" level=info msg="File /tmp/28077399.car successfully uploaded to IPFS with pin: QmUQ4XYJv9syrokUfUbhvA4bV8ce7w1Q2dF6NoNDfSDqxc" function=HandleObjectUploadToIPFS line=102 +time="2022-04-18T17:27:08Z" level=info msg="stream ids acked and trimmed: [1648848521276-0], for stream key: replication-1, with current length: 11698" function=processStream line=323 +``` + +If you see all of the above you're successfully running the full BSP pipeline. + +### Flag definitions + +`--mainnet` - lets geth know which network to synchronize with, and pull block specimen from, this can be `--ropsten`, `--goerli` , `--mainnet` etc + +`--port 0` - will auto-assign a port for geth to talk to other nodes in the network, but this may not work if you are behind a firewall. It would be better to explicitly assign a port and to ensure that port is open to any firewalls. + +`--http` - enables the json-rpc api over http + +`--log.debug` - enables a detailed log of the processes geth deals with going back and forth between + +`--syncmode` - this flag is used to enable different syncing strategies for geth and `full` and `snap` modes are supported for live block-specimen creation + +`--datadir` - specifies a local datadir path for geth (note we use “BSP” as the directory name with the Ethereum directory), this can be an pre-exisiting directory with chaindata for the given network flag synced to the most recent block. + +`--replication.targets` - this flag lets the BSP know where and how to send the BSP messages (this flag will not function without the usage of either one or both of the flags below, if both are selected a full block-replica is exported + +`--replica.result` - this flag lets the BSP know if all fields related to the block-result specification need to be exported (if only this flag is selected the exported object is a block-result) + +`--replica.specimen` - this flag lets the BSP know if all fields related to the block-specimen specification need to be exported (if only this flag is selected the exported object is a block-specimen) + +If both `--replica-result` & `--replica-specimen` are selected then a `block-replica` is exported containing all the fields for exporting any block fully along with its stored state. + +`--log.file` - specifies the file location where the log files have to be placed. In case of error (like permission errors), the logs are not recorded in files. + +## Go Ethereum Golang execution layer implementation of the Ethereum protocol. @@ -16,7 +378,7 @@ archives are published at https://geth.ethereum.org/downloads/. For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/getting-started/installing-geth). -Building `geth` requires both a Go (version 1.22 or later) and a C compiler. You can install +Building `geth` requires both a Go (version 1.19 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run ```shell @@ -89,7 +451,7 @@ This command will: This tool is optional and if you leave it out you can always attach it to an already running `geth` instance with `geth attach`. -### A Full node on the Holesky test network +### A Full node on the Görli test network Transitioning towards developers, if you'd like to play around with creating Ethereum contracts, you almost certainly would like to do that without any real money involved until @@ -98,23 +460,23 @@ network, you want to join the **test** network with your node, which is fully eq the main network, but with play-Ether only. ```shell -$ geth --holesky console +$ geth --goerli console ``` The `console` subcommand has the same meaning as above and is equally useful on the testnet too. -Specifying the `--holesky` flag, however, will reconfigure your `geth` instance a bit: +Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: - * Instead of connecting to the main Ethereum network, the client will connect to the Holesky + * Instead of connecting to the main Ethereum network, the client will connect to the Görli test network, which uses different P2P bootnodes, different network IDs and genesis states. * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` - will nest itself one level deeper into a `holesky` subfolder (`~/.ethereum/holesky` on + will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on Linux). Note, on OSX and Linux this also means that attaching to a running testnet node requires the use of a custom endpoint since `geth attach` will try to attach to a production node endpoint by default, e.g., - `geth attach /holesky/geth.ipc`. Windows users are not affected by + `geth attach /goerli/geth.ipc`. Windows users are not affected by this. *Note: Although some internal protective measures prevent transactions from diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml new file mode 100644 index 000000000..e25400536 --- /dev/null +++ b/docker-compose-ci.yml @@ -0,0 +1,119 @@ +version: '3' + +services: + redis: + image: redis:alpine + container_name: redis-srv + restart: always + expose: + - 6379 + environment: + - REDIS_REPLICATION_MODE=master + networks: + - cqt-net + entrypoint: redis-server #/usr/local/etc/redis/redis.conf + ports: + - "6379:6379" + + redis-commander: + image: rediscommander/redis-commander:latest + container_name: redis-commander-web + hostname: redis-commander + restart: always + depends_on: + - redis + environment: + - REDIS_HOSTS=local:redis:6379 + networks: + - cqt-net + ports: + - "8081:8081" + + node: + image: trufflesuite/ganache-cli:v6.12.2 + container_name: ganache-cli + restart: always + entrypoint: + - node + - /app/ganache-core.docker.cli.js + - --deterministic + - --db=/ganache_data + - --mnemonic + - 'minimum symptom minute gloom tragic situate silver mechanic salad amused elite beef' + - --networkId + - '5777' + - --hostname + - '0.0.0.0' + depends_on: + - redis-commander + networks: + - cqt-net + ports: + - "8545:8545" + + cqt-virtnet: + image: "ghcr.io/covalenthq/cqt-virtnet:latest" + container_name: proof-chain + restart: on-failure + expose: + - 8008 + entrypoint: > + /bin/bash -l -c " + truffle migrate --network docker; + nc -v agent 8008; + sleep 100000;" + depends_on: + - node + networks: + - cqt-net + environment: + npm_config_user: "root" + ports: + - "8008:8008" + + geth: + container_name: bsp-geth + build: + context: . + dockerfile: Dockerfile + restart: on-failure + depends_on: + cqt-virtnet: + condition: service_started + entrypoint: ["geth", "--mainnet", "--port", "0", "--log.debug", "--syncmode", "full", "--datadir", "/root/.ethereum/covalent", "--replication.targets", "redis://username:@redis:6379/0?topic=replication", "--replica.result", "--replica.specimen"] + networks: + - cqt-net + expose: + - 8545 + - 8546 + - 30303 + - 30303/udp + + agent: + image: "ghcr.io/covalenthq/bsp-agent:latest" + container_name: bsp-agent + restart: on-failure + depends_on: + cqt-virtnet: + condition: service_started + entrypoint: > + /bin/bash -l -c " + echo Waiting for proof-chain to be deployed...; + while ! nc -v -l -k -p 8008; + do + sleep 1; + done; + echo proof-chain contracts deployed!; + ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication#replicate --avro-codec-path=./codec/block-ethereum.avsc --binary-file-path=./bin/block-ethereum/ --replica-bucket=covalenthq-geth-block-specimen --segment-length=10 --proof-chain-address=0xEa2ff902dbeEECcc828757B881b343F9316752e5 --consumer-timeout=6000; + exit 0;" + environment: + - ETH_PRIVATE_KEY=${PRIVATE_KEY} + - ETH_RPC_URL=${RPC_URL} + - BLOCKCHAIN=${BLOCKCHAIN} + networks: + - cqt-net + ports: + - "8080:8080" + +networks: + cqt-net: diff --git a/docker-compose-testnet.yml b/docker-compose-testnet.yml new file mode 100644 index 000000000..189db0a59 --- /dev/null +++ b/docker-compose-testnet.yml @@ -0,0 +1,75 @@ +version: '3' + +services: + redis: + image: redis:alpine + container_name: redis-srv + restart: always + expose: + - 6379 + environment: + - REDIS_REPLICATION_MODE=master + networks: + - cqt-net + entrypoint: redis-server #/usr/local/etc/redis/redis.conf + ports: + - "6379:6379" + volumes: + - ./data/redis:/data + + redis-commander: + image: rediscommander/redis-commander:latest + container_name: redis-commander-web + hostname: redis-commander + restart: always + depends_on: + - redis + environment: + - REDIS_HOSTS=local:redis:6379 + networks: + - cqt-net + ports: + - "8081:8081" + + geth: + image: "ghcr.io/covalenthq/bsp-geth:latest" + container_name: bsp-geth + restart: always + depends_on: + - redis + entrypoint: ["geth", "--mainnet", "--port", "0", "--log.debug", "--syncmode", "full", "--http", "--datadir", "/data/.ethereum/covalent", "--allow-insecure-unlock", "--preload", "/scripts/geth-setup.js", "--replication.targets", "redis://username:@redis:6379/0?topic=replication", "--replica.result", "--replica.specimen"] + networks: + - cqt-net + expose: + - 8545 + - 8546 + - 30303 + - 30303/udp + volumes: + - ./scripts/:/scripts + - ./data/.ethereum/:/data/.ethereum/ + + agent: + image: "ghcr.io/covalenthq/bsp-agent:latest" + container_name: bsp-agent + restart: always + depends_on: + - geth + volumes: + - /Users/$USER/.config/gcloud:/app/gcloud + - ./data/bin/:/app/bin + entrypoint: > + /bin/bash -l -c " + ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication#replicate --avro-codec-path=./codec/block-ethereum.avsc --binary-file-path=./bin/block-ethereum/ --replica-bucket=covalenthq-geth-block-specimen --segment-length=1000 --proof-chain-address=0x67688076Da389cd1EcD90C4573d1D0e6BA04AC6a --consumer-timeout=6000 --gcp-svc-account=./gcloud/bsp-2.json; + exit 0;" + environment: + - ETH_PRIVATE_KEY=${PRIVATE_KEY} + - ETH_RPC_URL=${RPC_URL} + - BLOCKCHAIN=${BLOCKCHAIN} + networks: + - cqt-net + ports: + - "8080:8080" + +networks: + cqt-net: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 000000000..8bde1b91f --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,137 @@ +# Contributing + +* [Reporting Bugs](#bugs) +* [General Procedure](#general_procedure) + * [Development Procedure](#dev_procedure) + * [Dependencies](#dependencies) + * [Testing](#testing) + * [Branching Model and Release](#braching_model_and_release) + * [PR Targeting](#pr_targeting) + * [Pull Requests](#pull_requests) + * [Process for reviewing PRs](#reviewing_prs) + * [Pull Merge Procedure](#pull_merge_procedure) + + +## Reporting Bugs + +Please file bugs in the [GitHub Issue +Tracker](https://github.com/covalenthq/bsp-geth). Include at +least the following: + + - What happened + - What did you expect to happen instead of what *did* happen, if it's + not crazy obvious + - What operating system, operating system version and version of + go-ethereum you are running + - Console log entries, where possible and relevant + +If you're not sure whether something is relevant, erring on the side of +too much information will never be a cause for concern. + + +## General Procedure + +Contributing to this repo can mean many things such as participating in discussion or proposing code changes. To ensure a smooth workflow for all contributors, the following general procedure for contributing has been established: + +1. Either [open](https://github.com/covalenthq/bsp-geth/issues/new/choose) + or [find](https://github.com/covalenthq/bsp-geth/issues) an issue you have identified and would like to contribute to + resolving. + +2. Participate in thoughtful discussion on that issue. + +3. If you would like to contribute: + 1. If the issue is a proposal, ensure that the proposal has been accepted by the Covalent team. + 2. Ensure that nobody else has already begun working on the same issue. If someone already has, please make sure to contact the individual to collaborate. + 3. If nobody has been assigned the issue and you would like to work on it, make a comment on the issue to inform the + community of your intentions to begin work. Ideally, wait for confirmation that no one has started it. However, + if you are eager and do not get a prompt response, feel free to dive on in! + 4. Follow standard Github best practices: + 1. Fork the repo + 2. Branch from the HEAD of `main`(For core developers working within the go-ethereum repo, to ensure a clear ownership of branches, branches must be named with the convention `{moniker}/{issue#}-branch-name`). + 3. Make commits + 4. Submit a PR to `main` + 5. Be sure to submit the PR in `Draft` mode. Submit your PR early, even if it's incomplete as this indicates to the community you're working on something and allows them to provide comments early in the development process. + 6. When the code is complete it can be marked `Ready for Review`. + 7. Be sure to include a relevant change log entry in the `Unreleased` section of `CHANGELOG.md` (see file for log + format). + 8. Please make sure to run `gofmt` before every commit - the easiest way to do this is having your editor run it for you upon saving a file. Additionally, please ensure that your code is lint compliant by running `make lint` . There are CI tests built into the bsp-geth repository and all PR’s will require that these tests pass before they are able to be merged. + +**Note**: for very small or blatantly obvious problems (such as typos), it is not required to open an issue to submit a +PR, but be aware that for more complex problems/features, if a PR is opened before an adequate design discussion has +taken place in a github issue, that PR runs a high likelihood of being rejected. + +Looking for a good place to start contributing? How about checking out +some [good first issues](https://github.com/covalenthq/bsp-geth/issues). + +### Development Procedure + +1. The latest state of development is on `main`. +2. `main` must never + fail `make lint, make geth, make all ` +3. No `--force` onto `main` (except when reverting a broken commit, which should seldom happen). +4. Create your feature branch from `main` either on `github.com/covalenthq/bsp-geth`, or your fork ( + using `git remote add origin`). +5. Before submitting a pull request, begin `git rebase` on top of `main`. +6. Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) + guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). +7. Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) + guidelines. +8. Pull requests need to be based on and opened against the `main` branch. +9. Commit messages should be prefixed with the package(s) they modify. + * E.g. "eth, rpc: make trace configs optional" + + +### Dependencies + +We use [Go Modules](https://github.com/golang/go/wiki/Modules) to manage dependency versions. + +The main branch of every bsp-geth repository should just build with `go get`, which means they should be kept up-to-date +with their dependencies, so we can get away with telling people they can just `go get` our software. Since some dependencies are not under our control, a third party may break our build, in which case we can fall back +on `go mod tidy -v`. + +### Testing + +Covalent uses [GitHub Actions](https://github.com/features/actions) for automated [integration testing](https://github.com/covalenthq/bsp-geth/actions). + +### Branching Model and Release + +User-facing repos should adhere to the [trunk based development branching model](https://trunkbaseddevelopment.com/). + +Libraries need not follow the model strictly, but would be wise to. + +bsp-geth utilizes [semantic versioning](https://semver.org/). + +### PR Targeting + +Ensure that you base and target your PR on the `main` branch. + +All feature additions should be targeted against `main`. Bug fixes for an outstanding release candidate should be +targeted against the release candidate branch. + +### Pull Requests + +To accommodate the review process, we suggest that PRs are categorically broken up. Ideally each PR addresses only a +single issue. Additionally, as much as possible code refactoring and cleanup should be submitted as separate PRs from +bug fixes/feature-additions. + +### Process for reviewing PRs + +All PRs require at least 1 Review before merge. When reviewing PRs, please use the following review explanations: + +1. `LGTM` without an explicit approval means that the changes look good, but you haven't pulled down the code, run tests + locally and thoroughly reviewed it. +2. `Approval` through the GH UI means that you understand the code, documentation/spec is updated in the right places, + you have pulled down and tested the code locally. In addition: + * You must think through whether any added code could be partially combined (DRYed) with existing code. + * You must think through any potential security issues or incentive-compatibility flaws introduced by the changes. + * Naming convention must be consistent with the rest of the codebase. + * Code must live in a reasonable location, considering dependency structures (e.g. not importing testing modules in + production code, or including example code modules in production code). + * If you approve of the PR, you are responsible for fixing any of the issues mentioned here. +3. If you are only making "surface level" reviews, submit any notes as `Comments` without adding a review. + +### Pull Merge Procedure + +1. Ensure pull branch is rebased on `main`. +2. Ensure that all CI tests pass. +3. Squash merge pull request. \ No newline at end of file diff --git a/docs/arch.jpg b/docs/arch.jpg new file mode 100644 index 000000000..5c901ebf5 Binary files /dev/null and b/docs/arch.jpg differ diff --git a/docs/covalent.jpg b/docs/covalent.jpg new file mode 100644 index 000000000..54f33bdc7 Binary files /dev/null and b/docs/covalent.jpg differ diff --git a/docs/extract.png b/docs/extract.png new file mode 100644 index 000000000..5fc352d54 Binary files /dev/null and b/docs/extract.png differ diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md index 2e5c41c76..0986f00b6 100644 --- a/docs/postmortems/2021-08-22-split-postmortem.md +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -56,13 +56,13 @@ On the evening of 17th, we discussed options on how to handle it. We made a stat It was decided that in this specific instance, it would be possible to make a public announcement and a patch release: - The fix can be made pretty 'generically', e.g. always copying data on input to precompiles. -- The flaw is pretty difficult to find, given a generic fix in the call. The attacker needs to figure out that it concerns the precompiles, specifically the datacopy, and that it concerns the `RETURNDATA` buffer rather than the regular memory, and lastly the special circumstances to trigger it (overlapping but shifted input/output). +- The flaw is pretty difficult to find, given a generic fix in the call. The attacker needs to figure out that it concerns the precompiles, specifically the datcopy, and that it concerns the `RETURNDATA` buffer rather than the regular memory, and lastly the special circumstances to trigger it (overlapping but shifted input/output). Since we had merged the removal of `ETH65`, if the entire network were to upgrade, then nodes which have not yet implemented `ETH66` would be cut off from the network. After further discussions, we decided to: - Announce an upcoming security release on Tuesday (August 24th), via Twitter and official channels, plus reach out to downstream projects. - Temporarily revert the `ETH65`-removal. -- Place the fix into the PR optimizing the jumpdest analysis [23381](https://github.com/ethereum/go-ethereum/pull/23381). +- Place the fix into the PR optimizing the jumpdest analysis [233381](https://github.com/ethereum/go-ethereum/pull/23381). - After 4-8 weeks, release details about the vulnerability. diff --git a/docs/value.png b/docs/value.png new file mode 100644 index 000000000..bd8f6e09f Binary files /dev/null and b/docs/value.png differ diff --git a/params/version.go b/params/version.go index 6dd4468e9..1944f159e 100644 --- a/params/version.go +++ b/params/version.go @@ -28,8 +28,8 @@ const ( ) const ( - BspVersionMajor = 1 // Major version component of the current release - BspVersionMinor = 9 // Minor version component of the current release + BspVersionMajor = 2 // Major version component of the current release + BspVersionMinor = 0 // Minor version component of the current release BspVersionPatch = 0 // Patch version component of the current release )