diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c55c157d..da2a3d615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,31 @@ jobs: CARGO_INCREMENTAL: 1 RUST_BACKTRACE: 1 + test_zk_evm_proc_macro: + name: Test zk_evm_proc_macro + runs-on: ubuntu-latest + timeout-minutes: 30 + if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')" + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install nightly toolchain + uses: dtolnay/rust-toolchain@nightly + + - name: Set up rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Test in proc_macro subdirectory + run: cargo test --manifest-path proc_macro/Cargo.toml + env: + RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 + RUST_LOG: 1 + CARGO_INCREMENTAL: 1 + RUST_BACKTRACE: 1 + simple_proof_regular: name: Execute bash script to generate and verify a proof for a small block. runs-on: zero-ci diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml new file mode 100644 index 000000000..8d74a2fa4 --- /dev/null +++ b/.github/workflows/docker_build.yml @@ -0,0 +1,35 @@ +name: Docker Build & Run + +on: + push: + branches: [develop, main] + pull_request: + branches: + - "**" + workflow_dispatch: + branches: + - "**" + +jobs: + docker: + name: Build and run leader and worker docker images for regression check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build leader docker container + run: | + docker build --progress plain -t leader:${{ github.ref_name }} -f leader.Dockerfile . + + - name: Run leader docker container + run: | + docker run --rm leader:${{ github.ref_name }} --help + + - name: Build worker docker container + run: | + docker build --progress plain -t worker:${{ github.ref_name }} -f worker.Dockerfile . + + - name: Run worker docker container + run: | + docker run --rm worker:${{ github.ref_name }} --help diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml new file mode 100644 index 000000000..112ea158e --- /dev/null +++ b/.github/workflows/docker_build_push.yml @@ -0,0 +1,83 @@ +name: Docker Build & Push + +on: + push: + branches: [develop, main] + release: + types: [created] + +env: + REGISTRY: ghcr.io + IMAGE_NAME_LEADER: ${{ github.repository }}-leader + IMAGE_NAME_WORKER: ${{ github.repository }}-worker + +jobs: + docker: + name: Build and push leader and worker docker images to GitHub Container Registry + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Leader Docker + id: meta_leader + uses: docker/metadata-action@v5 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LEADER }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Push to GitHub Container Registry - Leader + uses: docker/build-push-action@v3 + with: + context: . + file: ./leader.Dockerfile + push: true + # platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta_leader.outputs.tags }} + labels: ${{ steps.meta_leader.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Extract metadata (tags, labels) for Worker Docker + id: meta_worker + uses: docker/metadata-action@v5 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_WORKER }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Push to GitHub Container Registry - Worker + uses: docker/build-push-action@v3 + with: + context: . + file: ./worker.Dockerfile + push: true + # platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta_worker.outputs.tags }} + labels: ${{ steps.meta_worker.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 5bed881a3..497a17208 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -1,22 +1,56 @@ name: PR check on: - pull_request_target: - types: - - opened - - edited - - synchronize + pull_request: + types: [opened, reopened, synchronize] + +permissions: + pull-requests: write jobs: - title: + pr_check: name: Validate PR runs-on: ubuntu-latest - if: ${{ - github.event.pull_request.author_association != 'CONTRIBUTOR' && - github.event.pull_request.author_association != 'MEMBER' && - ( - contains(fromJSON(secrets.RESTRICTED_KEYWORDS), github.event.pull_request.title) || - contains(fromJSON(secrets.RESTRICTED_KEYWORDS), github.event.pull_request.description - ) }} steps: - - run: gh pr close + - name: Set up keywords + id: setup_keywords + run: echo "RESTRICTED_KEYWORDS=$(echo '${{ secrets.RESTRICTED_KEYWORDS }}' | jq -r '.[]' | tr '\n' ' ')" >> $GITHUB_ENV + + - name: Check for spam PR + id: check + run: | + # Initialize variables to track spam presence + title_is_spam=false + description_is_spam=false + + # Check title for spam + for keyword in $RESTRICTED_KEYWORDS; do + if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then + title_is_spam=true + break + fi + done + + # Check description for spam + for keyword in $RESTRICTED_KEYWORDS; do + if echo "${{ github.event.pull_request.body }}" | grep -i -q "$keyword"; then + description_is_spam=true + break + fi + done + + # Set the output based on the presence of spam + if [ "$title_is_spam" = true ] || [ "$description_is_spam" = true ]; then + echo "is_spam=true" >> $GITHUB_ENV + else + echo "is_spam=false" >> $GITHUB_ENV + fi + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Close PR if spam are found and author is not a contributor or member + if: ${{ env.is_spam == 'true' && github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' }} + run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 76045dabc..eed9a7ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "futures-core", "futures-sink", @@ -31,7 +31,7 @@ dependencies = [ "actix-utils", "ahash", "base64 0.22.1", - "bitflags 2.5.0", + "bitflags 2.6.0", "brotli", "bytes", "bytestring", @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e9a1892803b02f53e25bea3e414ddd0501f12d97456c9d5ade4edf88f9516f" +checksum = "1752d7d62e2665da650a36d84abbf239f812534475d51f072a49a533513b7cdd" dependencies = [ "num_enum", "strum", @@ -405,9 +405,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b155716bab55763c95ba212806cf43d05bcc70e5f35b02bad20cf5ec7fe11fed" +checksum = "a43b18702501396fa9bcdeecd533bc85fac75150d308fc0f6800a01e6234a003" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8037e03c7f462a063f28daec9fda285a9a89da003c552f8637a80b9c8fd96241" +checksum = "d83524c1f6162fcb5b0decf775498a125066c86dda6066ed609531b0e912f85a" dependencies = [ "proc-macro2", "quote", @@ -1087,9 +1087,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-lc-rs" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7d844e282b4b56750b2d4e893b2205581ded8709fddd2b6aa5418c150ca877" +checksum = "a8a47f2fb521b70c11ce7369a6c5fa4bd6af7e5d62ec06303875bafe7c6ba245" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -1099,9 +1099,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a2c29203f6bf296d01141cc8bb9dbd5ecd4c27843f2ee0767bcd5985a927da" +checksum = "2927c7af777b460b7ccd95f8b67acd7b4c04ec8896bf0c8e80ba30523cffc057" dependencies = [ "bindgen", "cc", @@ -1125,7 +1125,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "itoa", "matchit", @@ -1226,7 +1226,7 @@ version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1266,9 +1266,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitvec" @@ -1413,9 +1413,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" dependencies = [ "jobserver", "libc", @@ -1502,9 +1502,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ "clap_builder", "clap_derive", @@ -1512,9 +1512,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ "anstream", "anstyle", @@ -1524,9 +1524,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -1573,6 +1573,14 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "compat" +version = "0.1.0" +dependencies = [ + "alloy", + "primitive-types 0.12.2", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2044,9 +2052,9 @@ dependencies = [ [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "elliptic-curve" @@ -2105,6 +2113,16 @@ dependencies = [ "syn 2.0.68", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -2118,6 +2136,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -2209,7 +2240,7 @@ dependencies = [ "anyhow", "bytes", "criterion", - "env_logger", + "env_logger 0.11.3", "ethereum-types", "hashbrown 0.14.5", "hex", @@ -2237,6 +2268,7 @@ dependencies = [ "starky", "static_assertions", "tiny-keccak", + "zk_evm_proc_macro", ] [[package]] @@ -2846,9 +2878,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" dependencies = [ "bytes", "futures-channel", @@ -2885,7 +2917,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "native-tls", "tokio", @@ -2895,16 +2927,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.3.1", + "hyper 1.4.0", "pin-project-lite", "socket2 0.5.7", "tokio", @@ -3372,9 +3404,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" @@ -3414,9 +3446,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -3470,7 +3502,6 @@ dependencies = [ "impl-serde", "keccak-hash 0.10.0", "log", - "num", "num-traits", "parking_lot", "pretty_env_logger", @@ -3536,9 +3567,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -3634,9 +3665,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "memchr", ] @@ -3668,7 +3699,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -4177,7 +4208,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" dependencies = [ - "env_logger", + "env_logger 0.10.2", "log", ] @@ -4260,7 +4291,6 @@ dependencies = [ name = "proof_gen" version = "0.2.0" dependencies = [ - "ethereum-types", "evm_arithmetization", "log", "paste", @@ -4276,7 +4306,7 @@ checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.5.0", + "bitflags 2.6.0", "lazy_static", "num-traits", "rand", @@ -4415,7 +4445,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -4524,7 +4554,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -4628,21 +4658,17 @@ dependencies = [ "anyhow", "chrono", "clap", + "compat", "evm_arithmetization", "futures", - "hex", - "hex-literal", - "itertools 0.13.0", "mpt_trie", "primitive-types 0.12.2", "prover", "serde", "serde_json", - "thiserror", "tokio", "tower", "trace_decoder", - "tracing", "tracing-subscriber", "url", "zero_bin_common", @@ -4743,7 +4769,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", @@ -4916,7 +4942,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -4979,9 +5005,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -5021,9 +5047,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.1" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +checksum = "079f3a42cd87588d924ed95b533f8d30a483388c4e400ab736a7058e34f16169" dependencies = [ "base64 0.22.1", "chrono", @@ -5039,9 +5065,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.1" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +checksum = "bc03aad67c1d26b7de277d51c86892e7d9a0110a2fe44bf6b26cc569fba302d6" dependencies = [ "darling", "proc-macro2", @@ -5146,25 +5172,11 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" name = "smt_trie" version = "0.1.0" dependencies = [ - "bytes", - "enum-as-inner", - "eth_trie", "ethereum-types", - "hex", "hex-literal", - "keccak-hash 0.10.0", - "log", - "num-traits", - "parking_lot", "plonky2", - "pretty_env_logger", "rand", - "rlp", - "rlp-derive", "serde", - "serde_json", - "thiserror", - "uint", ] [[package]] @@ -5486,9 +5498,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -5666,22 +5678,18 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" name = "trace_decoder" version = "0.4.0" dependencies = [ - "bytes", "ciborium", - "ciborium-io", "criterion", "enum-as-inner", "enumn", "ethereum-types", "evm_arithmetization", "hex", - "hex-literal", "keccak-hash 0.10.0", "log", "mpt_trie", "pretty_env_logger", "rlp", - "rlp-derive", "serde", "serde_json", "serde_with", @@ -5756,6 +5764,20 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "trybuild" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33a5f13f11071020bb12de7a16b925d2d58636175c20c11dc5f96cb64bb6c9b3" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "toml", +] + [[package]] name = "typenum" version = "1.17.0" @@ -5886,7 +5908,6 @@ dependencies = [ "clap", "dotenvy", "proof_gen", - "serde", "serde_json", "serde_path_to_error", "tracing", @@ -6279,7 +6300,6 @@ dependencies = [ "ops", "paladin-core", "tokio", - "tracing", "tracing-subscriber", "zero_bin_common", ] @@ -6380,6 +6400,16 @@ dependencies = [ "syn 2.0.68", ] +[[package]] +name = "zk_evm_proc_macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", + "trybuild", +] + [[package]] name = "zstd" version = "0.13.1" diff --git a/Cargo.toml b/Cargo.toml index c040e1c46..71298f0e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,15 @@ members = ["mpt_trie", "proof_gen", "trace_decoder", "evm_arithmetization", + "proc_macro", "zero_bin/leader", "zero_bin/worker", "zero_bin/common", "zero_bin/ops", "zero_bin/verifier", "zero_bin/rpc", - "zero_bin/prover", + "zero_bin/prover", + "compat", "zero_bin/coordinator"] resolver = "2" @@ -37,23 +39,25 @@ alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-featu "transport-http", "rpc-types-debug" ] } -anyhow = "1.0.40" +anyhow = "1.0.86" async-stream = "0.3.5" -axum = "0.7.4" -bytes = "1.5.0" -ciborium = "0.2.1" -ciborium-io = "0.2.1" -clap = { version = "4.4.6", features = ["derive", "env"] } +axum = "0.7.5" +bytes = "1.6.0" +ciborium = "0.2.2" +ciborium-io = "0.2.2" +clap = { version = "4.5.7", features = ["derive", "env"] } +compat = { path = "compat" } +__compat_primitive_types = { version = "0.12.2", package = "primitive-types" } criterion = "0.5.1" dotenvy = "0.15.7" enum-as-inner = "0.6.0" -enumn = "0.1.12" -env_logger = "0.10.0" +enumn = "0.1.13" +env_logger = "0.11.3" ethereum-types = "0.14.1" eth_trie = "0.4.0" evm_arithmetization = { path = "evm_arithmetization", version = "0.2.0" } -futures = "0.3.29" -hashbrown = "0.14.0" +futures = "0.3.30" +hashbrown = "0.14.5" hex = "0.4.3" hex-literal = "0.4.1" impl-codec = "0.6.0" @@ -62,17 +66,17 @@ impl-rlp = "0.3.0" impl-serde = "0.4.0" itertools = "0.13.0" keccak-hash = "0.10.0" -log = "0.4.20" +log = "0.4.21" mpt_trie = { path = "mpt_trie", version = "0.3.0" } -num = "0.4.1" -num-bigint = "0.4.3" +num = "0.4.3" +num-bigint = "0.4.5" num-traits = "0.2.19" -once_cell = "1.13.0" +once_cell = "1.19.0" paladin-core = "0.4.2" -parking_lot = "0.12.1" -paste = "1.0.14" -pest = "2.1.3" -pest_derive = "2.1.0" +parking_lot = "0.12.3" +paste = "1.0.15" +pest = "2.7.10" +pest_derive = "2.7.10" pretty_env_logger = "0.5.0" proof_gen = { path = "proof_gen", version = "0.2.0" } rand = "0.8.5" @@ -80,24 +84,24 @@ rand_chacha = "0.3.1" ripemd = "0.1.3" rlp = "0.5.2" rlp-derive = "0.1.0" -ruint = "1.12.1" -serde = "1.0.166" -serde_json = "1.0.96" -serde_path_to_error = "0.1.14" -serde_with = "3.4.0" +ruint = "1.12.3" +serde = "1.0.203" +serde_json = "1.0.118" +serde_path_to_error = "0.1.16" +serde_with = "3.8.1" smt_trie = { path = "smt_trie", version = "0.1.0" } -sha2 = "0.10.6" +sha2 = "0.10.8" static_assertions = "1.1.0" -thiserror = "1.0.49" +thiserror = "1.0.61" tiny-keccak = "2.0.2" -tokio = { version = "1.33.0", features = ["full"] } -toml = "0.8.12" +tokio = { version = "1.38.0", features = ["full"] } +toml = "0.8.14" tower = "0.4" trace_decoder = { path = "trace_decoder", version = "0.4.0" } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } uint = "0.9.5" -url = "2.5.0" +url = "2.5.2" # zero-bin related dependencies ops = { path = "zero_bin/ops" } @@ -111,6 +115,13 @@ plonky2_maybe_rayon = "0.2.0" plonky2_util = "0.2.0" starky = "0.4.0" +# proc macro related dependencies +proc-macro2 = "1.0" +quote = "1.0" +syn = "2.0" +trybuild = "1.0" +zk_evm_proc_macro = { path = "proc_macro" } + # GCP benchmark test dependencies google-cloud-storage = "0.16.0" google-cloud-googleapis = "0.12.0" diff --git a/compat/Cargo.toml b/compat/Cargo.toml new file mode 100644 index 000000000..69ff77eb7 --- /dev/null +++ b/compat/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "compat" +version = "0.1.0" +publish = false # TODO(): https://github.com/0xPolygonZero/zk_evm/issues/314 find a better place for this +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +alloy = {workspace = true } +__compat_primitive_types = { workspace = true } diff --git a/zero_bin/rpc/src/compat.rs b/compat/src/lib.rs similarity index 95% rename from zero_bin/rpc/src/compat.rs rename to compat/src/lib.rs index 7fb0a1563..e83eb6eaf 100644 --- a/zero_bin/rpc/src/compat.rs +++ b/compat/src/lib.rs @@ -1,4 +1,6 @@ +/// A trait to convert between alloy and ethereum types. pub trait Compat { + /// Convert the type to another type fn compat(self) -> Out; } diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index 9ae67e0f0..8d98c214b 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -45,6 +45,7 @@ serde_json = { workspace = true } # Local dependencies mpt_trie = { workspace = true } +zk_evm_proc_macro = { workspace = true } [dev-dependencies] criterion = { workspace = true } diff --git a/evm_arithmetization/src/cpu/columns/mod.rs b/evm_arithmetization/src/cpu/columns/mod.rs index d4abd71d9..661c4d18f 100644 --- a/evm_arithmetization/src/cpu/columns/mod.rs +++ b/evm_arithmetization/src/cpu/columns/mod.rs @@ -4,12 +4,13 @@ use core::mem::{size_of, transmute}; use core::ops::{Index, IndexMut}; use plonky2::field::types::Field; +use zk_evm_proc_macro::Columns; use crate::cpu::columns::general::CpuGeneralColumnsView; use crate::cpu::columns::ops::OpsColumnsView; use crate::cpu::membus::NUM_GP_CHANNELS; use crate::memory; -use crate::util::{indices_arr, transmute_no_compile_time_size_checks}; +use crate::util::indices_arr; mod general; /// Cpu operation flags. @@ -54,7 +55,7 @@ pub(crate) struct PartialMemoryChannelView { } #[repr(C)] -#[derive(Clone, Copy, Eq, PartialEq, Debug)] +#[derive(Columns, Clone, Copy, Eq, PartialEq, Debug)] pub(crate) struct CpuColumnsView { /// If CPU cycle: Current context. pub context: T, @@ -98,70 +99,6 @@ pub(crate) struct CpuColumnsView { /// `u8` is guaranteed to have a `size_of` of 1. pub(crate) const NUM_CPU_COLUMNS: usize = size_of::>(); -impl Default for CpuColumnsView { - fn default() -> Self { - Self::from([F::ZERO; NUM_CPU_COLUMNS]) - } -} - -impl From<[T; NUM_CPU_COLUMNS]> for CpuColumnsView { - fn from(value: [T; NUM_CPU_COLUMNS]) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl From> for [T; NUM_CPU_COLUMNS] { - fn from(value: CpuColumnsView) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl Borrow> for [T; NUM_CPU_COLUMNS] { - fn borrow(&self) -> &CpuColumnsView { - unsafe { transmute(self) } - } -} - -impl BorrowMut> for [T; NUM_CPU_COLUMNS] { - fn borrow_mut(&mut self) -> &mut CpuColumnsView { - unsafe { transmute(self) } - } -} - -impl Borrow<[T; NUM_CPU_COLUMNS]> for CpuColumnsView { - fn borrow(&self) -> &[T; NUM_CPU_COLUMNS] { - unsafe { transmute(self) } - } -} - -impl BorrowMut<[T; NUM_CPU_COLUMNS]> for CpuColumnsView { - fn borrow_mut(&mut self) -> &mut [T; NUM_CPU_COLUMNS] { - unsafe { transmute(self) } - } -} - -impl Index for CpuColumnsView -where - [T]: Index, -{ - type Output = <[T] as Index>::Output; - - fn index(&self, index: I) -> &Self::Output { - let arr: &[T; NUM_CPU_COLUMNS] = self.borrow(); - <[T] as Index>::index(arr, index) - } -} - -impl IndexMut for CpuColumnsView -where - [T]: IndexMut, -{ - fn index_mut(&mut self, index: I) -> &mut Self::Output { - let arr: &mut [T; NUM_CPU_COLUMNS] = self.borrow_mut(); - <[T] as IndexMut>::index_mut(arr, index) - } -} - const fn make_col_map() -> CpuColumnsView { let indices_arr = indices_arr::(); unsafe { transmute::<[usize; NUM_CPU_COLUMNS], CpuColumnsView>(indices_arr) } diff --git a/evm_arithmetization/src/cpu/columns/ops.rs b/evm_arithmetization/src/cpu/columns/ops.rs index c15d65722..9963215f6 100644 --- a/evm_arithmetization/src/cpu/columns/ops.rs +++ b/evm_arithmetization/src/cpu/columns/ops.rs @@ -2,11 +2,11 @@ use core::borrow::{Borrow, BorrowMut}; use core::mem::{size_of, transmute}; use core::ops::{Deref, DerefMut}; -use crate::util::transmute_no_compile_time_size_checks; +use zk_evm_proc_macro::{Columns, DerefColumns}; /// Structure representing the flags for the various opcodes. #[repr(C)] -#[derive(Clone, Copy, Eq, PartialEq, Debug)] +#[derive(Columns, DerefColumns, Clone, Copy, Eq, PartialEq, Debug)] pub(crate) struct OpsColumnsView { /// Combines ADD, MUL, SUB, DIV, MOD, LT, GT and BYTE flags. pub binary_op: T, @@ -50,40 +50,3 @@ pub(crate) struct OpsColumnsView { /// Number of columns in Cpu Stark. /// `u8` is guaranteed to have a `size_of` of 1. pub(crate) const NUM_OPS_COLUMNS: usize = size_of::>(); - -impl From<[T; NUM_OPS_COLUMNS]> for OpsColumnsView { - fn from(value: [T; NUM_OPS_COLUMNS]) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl From> for [T; NUM_OPS_COLUMNS] { - fn from(value: OpsColumnsView) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl Borrow> for [T; NUM_OPS_COLUMNS] { - fn borrow(&self) -> &OpsColumnsView { - unsafe { transmute(self) } - } -} - -impl BorrowMut> for [T; NUM_OPS_COLUMNS] { - fn borrow_mut(&mut self) -> &mut OpsColumnsView { - unsafe { transmute(self) } - } -} - -impl Deref for OpsColumnsView { - type Target = [T; NUM_OPS_COLUMNS]; - fn deref(&self) -> &Self::Target { - unsafe { transmute(self) } - } -} - -impl DerefMut for OpsColumnsView { - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { transmute(self) } - } -} diff --git a/evm_arithmetization/src/cpu/kernel/asm/core/transfer.asm b/evm_arithmetization/src/cpu/kernel/asm/core/transfer.asm index 6960bf07e..986c4f991 100644 --- a/evm_arithmetization/src/cpu/kernel/asm/core/transfer.asm +++ b/evm_arithmetization/src/cpu/kernel/asm/core/transfer.asm @@ -63,10 +63,11 @@ global deduct_eth_insufficient_balance: // Pre stack: addr, amount, redest // Post stack: (empty) global add_eth: + // stack: addr, amount, retdest + DUP1 %insert_touched_addresses // stack: addr, amount, retdest DUP2 ISZERO %jumpi(add_eth_zero_amount) // stack: addr, amount, retdest - DUP1 %insert_touched_addresses DUP1 %mpt_read_state_trie // stack: account_ptr, addr, amount, retdest DUP1 ISZERO %jumpi(add_eth_new_account) // If the account pointer is null, we need to create the account. diff --git a/evm_arithmetization/src/cpu/kernel/asm/memory/memset.asm b/evm_arithmetization/src/cpu/kernel/asm/memory/memset.asm index 792aeabc6..2cd50e85a 100644 --- a/evm_arithmetization/src/cpu/kernel/asm/memory/memset.asm +++ b/evm_arithmetization/src/cpu/kernel/asm/memory/memset.asm @@ -32,13 +32,12 @@ memset_finish: %jumpi(memset_bytes_empty) // stack: DST, final_count, retdest - DUP2 PUSH 0 - DUP3 - // stack: DST, 0, final_count, DST, final_count, retdest + SWAP1 + // stack: DST, 0, final_count, retdest %mstore_unpacking - // stack: DST, final_count, retdest - %pop3 + // stack: DST', retdest + POP // stack: retdest JUMP diff --git a/evm_arithmetization/src/keccak_sponge/columns.rs b/evm_arithmetization/src/keccak_sponge/columns.rs index 44c707bf2..ce35c0867 100644 --- a/evm_arithmetization/src/keccak_sponge/columns.rs +++ b/evm_arithmetization/src/keccak_sponge/columns.rs @@ -2,7 +2,9 @@ use core::borrow::{Borrow, BorrowMut}; use core::mem::{size_of, transmute}; use core::ops::Range; -use crate::util::{indices_arr, transmute_no_compile_time_size_checks}; +use zk_evm_proc_macro::Columns; + +use crate::util::indices_arr; /// Total number of sponge bytes: number of rate bytes + number of capacity /// bytes. @@ -27,7 +29,7 @@ pub(crate) const KECCAK_DIGEST_U32S: usize = KECCAK_DIGEST_BYTES / 4; /// A view of `KeccakSpongeStark`'s columns. #[repr(C)] -#[derive(Eq, PartialEq, Debug)] +#[derive(Columns, Eq, PartialEq, Debug)] pub(crate) struct KeccakSpongeColumnsView { /// 1 if this row represents a full input block, i.e. one in which each byte /// is an input byte, not a padding byte; 0 otherwise. @@ -113,48 +115,6 @@ pub(crate) const fn get_single_block_bytes_value(i: usize) -> usize { get_block_bytes_range().start + i } -impl From<[T; NUM_KECCAK_SPONGE_COLUMNS]> for KeccakSpongeColumnsView { - fn from(value: [T; NUM_KECCAK_SPONGE_COLUMNS]) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl From> for [T; NUM_KECCAK_SPONGE_COLUMNS] { - fn from(value: KeccakSpongeColumnsView) -> Self { - unsafe { transmute_no_compile_time_size_checks(value) } - } -} - -impl Borrow> for [T; NUM_KECCAK_SPONGE_COLUMNS] { - fn borrow(&self) -> &KeccakSpongeColumnsView { - unsafe { transmute(self) } - } -} - -impl BorrowMut> for [T; NUM_KECCAK_SPONGE_COLUMNS] { - fn borrow_mut(&mut self) -> &mut KeccakSpongeColumnsView { - unsafe { transmute(self) } - } -} - -impl Borrow<[T; NUM_KECCAK_SPONGE_COLUMNS]> for KeccakSpongeColumnsView { - fn borrow(&self) -> &[T; NUM_KECCAK_SPONGE_COLUMNS] { - unsafe { transmute(self) } - } -} - -impl BorrowMut<[T; NUM_KECCAK_SPONGE_COLUMNS]> for KeccakSpongeColumnsView { - fn borrow_mut(&mut self) -> &mut [T; NUM_KECCAK_SPONGE_COLUMNS] { - unsafe { transmute(self) } - } -} - -impl Default for KeccakSpongeColumnsView { - fn default() -> Self { - [T::default(); NUM_KECCAK_SPONGE_COLUMNS].into() - } -} - const fn make_col_map() -> KeccakSpongeColumnsView { let indices_arr = indices_arr::(); unsafe { diff --git a/evm_arithmetization/src/util.rs b/evm_arithmetization/src/util.rs index 15a93c0af..373339cdd 100644 --- a/evm_arithmetization/src/util.rs +++ b/evm_arithmetization/src/util.rs @@ -147,14 +147,6 @@ pub(crate) const fn indices_arr() -> [usize; N] { indices_arr } -pub(crate) unsafe fn transmute_no_compile_time_size_checks(value: T) -> U { - debug_assert_eq!(size_of::(), size_of::()); - // Need ManuallyDrop so that `value` is not dropped by this function. - let value = ManuallyDrop::new(value); - // Copy the bit pattern. The original value is no longer safe to use. - transmute_copy(&value) -} - pub(crate) fn addmod(x: U256, y: U256, m: U256) -> U256 { if m.is_zero() { return m; diff --git a/leader.Dockerfile b/leader.Dockerfile new file mode 100644 index 000000000..ef69a430f --- /dev/null +++ b/leader.Dockerfile @@ -0,0 +1,60 @@ +FROM rustlang/rust:nightly-bullseye-slim as builder + +RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config + +RUN mkdir -p zero_bin +COPY Cargo.toml . +# Cleanup all workspace members and add selected crates again +RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml +RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/leader", "zero_bin\/prover", "zero_bin\/rpc", "zero_bin\/common", \ + "zero_bin\/ops"\, "evm_arithmetization", "trace_decoder", "mpt_trie", "proc_macro", "compat"\]#' Cargo.toml +COPY Cargo.lock . +COPY ./rust-toolchain.toml ./ +RUN cat ./Cargo.toml +COPY ./.env ./.env + +COPY proof_gen proof_gen +COPY mpt_trie mpt_trie +COPY proc_macro proc_macro +COPY compat compat +COPY trace_decoder trace_decoder +COPY evm_arithmetization evm_arithmetization +COPY zero_bin/common zero_bin/common +COPY zero_bin/ops zero_bin/ops +COPY zero_bin/rpc zero_bin/rpc +COPY zero_bin/prover zero_bin/prover +COPY zero_bin/leader zero_bin/leader + + +RUN \ + touch zero_bin/common/src/lib.rs && \ + touch zero_bin/ops/src/lib.rs && \ + touch zero_bin/leader/src/main.rs && \ + touch zero_bin/rpc/src/lib.rs && \ + touch zero_bin/prover/src/lib.rs && \ + touch evm_arithmetization/src/lib.rs && \ + touch trace_decoder/src/lib.rs && \ + touch mpt_trie/src/lib.rs && \ + touch proc_macro/src/lib.rs && \ + touch compat/src/lib.rs + +# Disable the lld linker for now, as it's causing issues with the linkme package. +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' + +RUN cargo build --release --bin leader +RUN cargo build --release --bin rpc + + +FROM debian:bullseye-slim +RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 +COPY --from=builder ./target/release/leader /usr/local/bin/leader +COPY --from=builder ./target/release/rpc /usr/local/bin/rpc +COPY --from=builder ./.env /.env + +# Workaround for the issue with the Cargo.lock search path +# Related to issue https://github.com/0xPolygonZero/zk_evm/issues/311 +RUN mkdir -p zero_bin/leader + +ENTRYPOINT ["/usr/local/bin/leader"] diff --git a/mpt_trie/Cargo.toml b/mpt_trie/Cargo.toml index a2038fc86..46a14df26 100644 --- a/mpt_trie/Cargo.toml +++ b/mpt_trie/Cargo.toml @@ -22,7 +22,6 @@ keccak-hash = { workspace = true } parking_lot = { workspace = true, features = ["serde"] } thiserror = { workspace = true } log = { workspace = true } -num = { workspace = true, optional = true } num-traits = { workspace = true } uint = { workspace = true } rlp = { workspace = true } @@ -41,7 +40,7 @@ serde_json = { workspace = true } [features] default = ["trie_debug"] -trie_debug = ["num"] +trie_debug = [] [lib] doc-scrape-examples = true diff --git a/proc_macro/Cargo.toml b/proc_macro/Cargo.toml new file mode 100644 index 000000000..d2deca37f --- /dev/null +++ b/proc_macro/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "zk_evm_proc_macro" +version = "0.1.0" +publish = false +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +proc-macro = true + +[[test]] +name = "tests" +path = "tests/compiletest.rs" + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } + +[dev-dependencies] +trybuild = { workspace = true } diff --git a/proc_macro/src/common.rs b/proc_macro/src/common.rs new file mode 100644 index 000000000..b8c96ed92 --- /dev/null +++ b/proc_macro/src/common.rs @@ -0,0 +1,41 @@ +use syn::punctuated::Punctuated; +use syn::{token, Attribute, Meta}; + +/// Prefixes an error message and generates a `syn::Error` from the message. +macro_rules! span_err { + ($ast:expr, $msg:literal $(,)?) => { + ::syn::Error::new_spanned($ast, ::core::concat!("zk_evm_proc_macro error: ", $msg)) + }; +} +pub(crate) use span_err; + +/// Checks the condition and returns early with a prefixed error message if +/// false. +macro_rules! ensure { + ($cond:expr, $ast:expr, $msg:literal $(,)?) => { + if !$cond { + return Err($crate::common::span_err!($ast, $msg)); + } + }; +} +pub(crate) use ensure; + +/// Parses the `Meta` of a `repr` attribute and returns true if one of the +/// elements is "C". +fn is_meta_c(outer: &Meta) -> bool { + if let Meta::List(inner) = outer { + let parsed: Punctuated = inner + .parse_args_with(Punctuated::parse_terminated) + .unwrap_or_default(); + parsed.iter().any(|meta| meta.path().is_ident("C")) + } else { + false + } +} + +/// Returns true if `#[repr(C)]` is contained in the attributes. +pub(crate) fn is_repr_c<'a>(attrs: impl IntoIterator) -> bool { + attrs + .into_iter() + .any(|attr| attr.path().is_ident("repr") && is_meta_c(&attr.meta)) +} diff --git a/proc_macro/src/impls/columns.rs b/proc_macro/src/impls/columns.rs new file mode 100644 index 000000000..f4f42a1d8 --- /dev/null +++ b/proc_macro/src/impls/columns.rs @@ -0,0 +1,133 @@ +use quote::quote; +use syn::{Data, DeriveInput, Result}; + +use crate::common::{ensure, is_repr_c}; + +/// Implements `Borrow`, `BorrowMut`, `From`, `Index`, `IndexMut`, and +/// `Default`. +pub(crate) fn try_derive(ast: DeriveInput) -> Result { + let is_struct = matches!(ast.data, Data::Struct(_)); + ensure!(is_struct, &ast, "expected `struct`"); + + // Check that the struct is `#[repr(C)]`. + let repr_c = is_repr_c(&ast.attrs); + ensure!(repr_c, &ast, "column struct must be `#[repr(C)]`"); + + // The name of the struct. + let name = &ast.ident; + + // SAFETY: `u8` is guaranteed to have a `size_of` of 1. + // https://doc.rust-lang.org/reference/type-layout.html#primitive-data-layout + let num_columns = quote!(::core::mem::size_of::<#name>()); + + // SAFETY: A struct generic over T has the same layout as an array [T; N] if: + // - The struct is `#[repr(C)]`. + // - Every field is one of T, [T; M], or a type with the same layout as [T; M], + // - The total number of elements of type T is N. + // https://doc.rust-lang.org/reference/type-layout.html#reprc-structs + // https://doc.rust-lang.org/reference/type-layout.html#array-layout + Ok(quote! { + impl ::core::borrow::Borrow<#name> for [T; #num_columns] + where + T: ::core::marker::Copy, + { + fn borrow(&self) -> &#name { + unsafe { ::core::mem::transmute(self) } + } + } + + impl ::core::borrow::BorrowMut<#name> for [T; #num_columns] + where + T: ::core::marker::Copy, + { + fn borrow_mut(&mut self) -> &mut #name { + unsafe { ::core::mem::transmute(self) } + } + } + + impl ::core::borrow::Borrow<[T; #num_columns]> for #name + where + T: ::core::marker::Copy, + { + fn borrow(&self) -> &[T; #num_columns] { + unsafe { ::core::mem::transmute(self) } + } + } + + impl ::core::borrow::BorrowMut<[T; #num_columns]> for #name + where + T: ::core::marker::Copy, + { + fn borrow_mut(&mut self) -> &mut [T; #num_columns] { + unsafe { ::core::mem::transmute(self) } + } + } + + impl From<[T; #num_columns]> for #name + where + T: ::core::marker::Copy, + { + fn from(value: [T; #num_columns]) -> Self { + debug_assert_eq!( + ::core::mem::size_of::<#name>(), + ::core::mem::size_of::<[T; #num_columns]>() + ); + // Need ManuallyDrop so that `value` is not dropped by this function. + let value = ::core::mem::ManuallyDrop::new(value); + // Copy the bit pattern. The original value is no longer safe to use. + unsafe { ::core::mem::transmute_copy(&value) } + } + } + + impl From<#name> for [T; #num_columns] + where + T: ::core::marker::Copy, + { + fn from(value: #name) -> Self { + debug_assert_eq!( + ::core::mem::size_of::<#name>(), + ::core::mem::size_of::<[T; #num_columns]>() + ); + // Need ManuallyDrop so that `value` is not dropped by this function. + let value = ::core::mem::ManuallyDrop::new(value); + // Copy the bit pattern. The original value is no longer safe to use. + unsafe { ::core::mem::transmute_copy(&value) } + } + } + + impl ::core::ops::Index for #name + where + T: ::core::marker::Copy, + [T]: ::core::ops::Index, + { + type Output = <[T] as ::core::ops::Index>::Output; + + fn index(&self, index: I) -> &>::Output { + let arr = ::core::borrow::Borrow::<[T; #num_columns]>::borrow(self); + <[T] as ::core::ops::Index>::index(arr, index) + } + } + + impl ::core::ops::IndexMut for #name + where + T: ::core::marker::Copy, + [T]: ::core::ops::IndexMut, + { + fn index_mut(&mut self, index: I) -> &mut >::Output { + let arr = ::core::borrow::BorrowMut::<[T; #num_columns]>::borrow_mut(self); + <[T] as ::core::ops::IndexMut>::index_mut(arr, index) + } + } + + impl ::core::default::Default for #name + where + T: ::core::marker::Copy + ::core::default::Default, + { + fn default() -> Self { + ::core::convert::Into::::into( + [::default(); #num_columns] + ) + } + } + }) +} diff --git a/proc_macro/src/impls/deref_columns.rs b/proc_macro/src/impls/deref_columns.rs new file mode 100644 index 000000000..534ab09c2 --- /dev/null +++ b/proc_macro/src/impls/deref_columns.rs @@ -0,0 +1,43 @@ +use quote::quote; +use syn::{Data, DeriveInput, Result}; + +use crate::common::{ensure, is_repr_c}; + +/// Implements `Deref` and `DerefMut`. +pub(crate) fn try_derive(ast: DeriveInput) -> Result { + let is_struct = matches!(ast.data, Data::Struct(_)); + ensure!(is_struct, &ast, "expected `struct`"); + + // Check that the struct is `#[repr(C)]`. + let repr_c = is_repr_c(&ast.attrs); + ensure!(repr_c, &ast, "column struct must be `#[repr(C)]`"); + + // The name of the struct. + let name = &ast.ident; + + // SAFETY: `u8` is guaranteed to have a `size_of` of 1. + // https://doc.rust-lang.org/reference/type-layout.html#primitive-data-layout + let num_columns = quote!(::core::mem::size_of::<#name>()); + + // SAFETY: A struct generic over T has the same layout as an array [T; N] if: + // - The struct is `#[repr(C)]`. + // - Every field is one of T, [T; M], or a type with the same layout as [T; M], + // - The total number of elements of type T is N. + // https://doc.rust-lang.org/reference/type-layout.html#reprc-structs + // https://doc.rust-lang.org/reference/type-layout.html#array-layout + Ok(quote! { + impl ::core::ops::Deref for #name { + type Target = [T; #num_columns]; + + fn deref(&self) -> &::Target { + unsafe { ::core::mem::transmute(self) } + } + } + + impl ::core::ops::DerefMut for #name { + fn deref_mut(&mut self) -> &mut ::Target { + unsafe { ::core::mem::transmute(self) } + } + } + }) +} diff --git a/proc_macro/src/impls/mod.rs b/proc_macro/src/impls/mod.rs new file mode 100644 index 000000000..2e0de48ac --- /dev/null +++ b/proc_macro/src/impls/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod columns; +pub(crate) mod deref_columns; diff --git a/proc_macro/src/lib.rs b/proc_macro/src/lib.rs new file mode 100644 index 000000000..d0a9d8225 --- /dev/null +++ b/proc_macro/src/lib.rs @@ -0,0 +1,47 @@ +//! This library provides two convenient derive macros for interpreting arrays +//! of field elements as structs representing an AIR. +//! +//! Deriving [`Columns`] on a struct `Struct` implements the following +//! conversion traits between `Struct` and arrays `[T; N]` where `N` is the +//! number of fields in the struct: [`Borrow`], [`BorrowMut`], and [`From`]. +//! Additionally, the traits [`Index`], [`IndexMut`], and [`Default`] are +//! implemented for `Struct`. +//! +//! Deriving [`DerefColumns`] for a struct generic over `T` implements [`Deref`] +//! and [`DerefMut`] with target `[T; N]` where `N` is the number of fields in +//! the struct. +//! +//! These implementations employ unsafe code and place a burden on the user to +//! ensure their safe usage. Please see the respective macro implementations to +//! understand the conditions that should be upheld by any struct deriving +//! [`Columns`] or [`DerefColumns`]. In short, the struct must be `#[repr(C)]` +//! and all fields must be one of `T`, `[T; M]`, or a type with the same layout +//! as `[T; M]`. +//! +//! [`Borrow`]: ::core::borrow::Borrow +//! [`BorrowMut`]: ::core::borrow::BorrowMut +//! [`Index`]: ::core::ops::Index +//! [`IndexMut`]: ::core::ops::IndexMut +//! [`Deref`]: ::core::ops::Deref +//! [`DerefMut`]: ::core::ops::DerefMut + +pub(crate) mod common; +mod impls; + +use impls::{columns, deref_columns}; + +#[proc_macro_derive(Columns)] +pub fn derive_columns(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(input as syn::DeriveInput); + columns::try_derive(ast) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} + +#[proc_macro_derive(DerefColumns)] +pub fn derive_deref_columns(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(input as syn::DeriveInput); + deref_columns::try_derive(ast) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} diff --git a/proc_macro/tests/compile/mod.rs b/proc_macro/tests/compile/mod.rs new file mode 100644 index 000000000..838146ec9 --- /dev/null +++ b/proc_macro/tests/compile/mod.rs @@ -0,0 +1,26 @@ +mod test_impls; + +use zk_evm_proc_macro::{Columns, DerefColumns}; + +#[repr(C)] +pub struct NestedColumns { + x: T, + y: [T; 3], +} + +#[repr(C)] +#[derive(DerefColumns)] +pub struct OpColumns { + is_op_a: T, + is_op_b: T, + is_op_c: T, +} + +#[repr(C)] +#[derive(Columns)] +pub struct AllColumns { + a: T, + b: [T; 4], + c: [NestedColumns; 5], + op: OpColumns, +} diff --git a/proc_macro/tests/compile/test_impls.rs b/proc_macro/tests/compile/test_impls.rs new file mode 100644 index 000000000..69e0c28db --- /dev/null +++ b/proc_macro/tests/compile/test_impls.rs @@ -0,0 +1,34 @@ +use core::borrow::{Borrow, BorrowMut}; +use core::ops::{Deref, DerefMut, Index, IndexMut}; + +use super::*; + +const NUM_OP_COLUMNS: usize = core::mem::size_of::>(); + +#[allow(unused)] +trait TestDerefColumns: Deref + DerefMut {} +impl TestDerefColumns for OpColumns {} + +const NUM_COLUMNS: usize = core::mem::size_of::>(); + +#[allow(unused)] +trait TestColumns: + Borrow<[T; NUM_COLUMNS]> + + BorrowMut<[T; NUM_COLUMNS]> + + From<[T; NUM_COLUMNS]> + + Index>::Output> + + IndexMut + + Default +where + [T]: Index + IndexMut, + [T; NUM_COLUMNS]: Borrow, + [T; NUM_COLUMNS]: BorrowMut, + [T; NUM_COLUMNS]: From, +{ +} +impl TestColumns for AllColumns +where + T: Copy + Default, + [T]: Index + IndexMut, +{ +} diff --git a/proc_macro/tests/compile_fail/not_repr_c.rs b/proc_macro/tests/compile_fail/not_repr_c.rs new file mode 100644 index 000000000..c3d8ed11b --- /dev/null +++ b/proc_macro/tests/compile_fail/not_repr_c.rs @@ -0,0 +1,9 @@ +use zk_evm_proc_macro::Columns; + +#[derive(Columns)] +struct Columns { + a: T, + b: [T; 3], +} + +fn main() {} diff --git a/proc_macro/tests/compile_fail/not_repr_c.stderr b/proc_macro/tests/compile_fail/not_repr_c.stderr new file mode 100644 index 000000000..71e4cc26b --- /dev/null +++ b/proc_macro/tests/compile_fail/not_repr_c.stderr @@ -0,0 +1,8 @@ +error: zk_evm_proc_macro error: column struct must be `#[repr(C)]` + --> tests/compile_fail/not_repr_c.rs:4:1 + | +4 | / struct Columns { +5 | | a: T, +6 | | b: [T; 3], +7 | | } + | |_^ diff --git a/proc_macro/tests/compile_fail/not_repr_c_deref.rs b/proc_macro/tests/compile_fail/not_repr_c_deref.rs new file mode 100644 index 000000000..cac29f0df --- /dev/null +++ b/proc_macro/tests/compile_fail/not_repr_c_deref.rs @@ -0,0 +1,9 @@ +use zk_evm_proc_macro::DerefColumns; + +#[derive(DerefColumns)] +struct Columns { + a: T, + b: [T; 3], +} + +fn main() {} diff --git a/proc_macro/tests/compile_fail/not_repr_c_deref.stderr b/proc_macro/tests/compile_fail/not_repr_c_deref.stderr new file mode 100644 index 000000000..26ed6ba0f --- /dev/null +++ b/proc_macro/tests/compile_fail/not_repr_c_deref.stderr @@ -0,0 +1,8 @@ +error: zk_evm_proc_macro error: column struct must be `#[repr(C)]` + --> tests/compile_fail/not_repr_c_deref.rs:4:1 + | +4 | / struct Columns { +5 | | a: T, +6 | | b: [T; 3], +7 | | } + | |_^ diff --git a/proc_macro/tests/compile_fail/not_struct.rs b/proc_macro/tests/compile_fail/not_struct.rs new file mode 100644 index 000000000..42be545a4 --- /dev/null +++ b/proc_macro/tests/compile_fail/not_struct.rs @@ -0,0 +1,10 @@ +use zk_evm_proc_macro::Columns; + +#[repr(C)] +#[derive(Columns)] +enum Maybe { + Nothing, + Just(T), +} + +fn main() {} diff --git a/proc_macro/tests/compile_fail/not_struct.stderr b/proc_macro/tests/compile_fail/not_struct.stderr new file mode 100644 index 000000000..b7a18ddd7 --- /dev/null +++ b/proc_macro/tests/compile_fail/not_struct.stderr @@ -0,0 +1,10 @@ +error: zk_evm_proc_macro error: expected `struct` + --> tests/compile_fail/not_struct.rs:3:1 + | +3 | / #[repr(C)] +4 | | #[derive(Columns)] +5 | | enum Maybe { +6 | | Nothing, +7 | | Just(T), +8 | | } + | |_^ diff --git a/proc_macro/tests/compile_fail/not_struct_deref.rs b/proc_macro/tests/compile_fail/not_struct_deref.rs new file mode 100644 index 000000000..be429e6b2 --- /dev/null +++ b/proc_macro/tests/compile_fail/not_struct_deref.rs @@ -0,0 +1,10 @@ +use zk_evm_proc_macro::DerefColumns; + +#[repr(C)] +#[derive(DerefColumns)] +enum Maybe { + Nothing, + Just(T), +} + +fn main() {} diff --git a/proc_macro/tests/compile_fail/not_struct_deref.stderr b/proc_macro/tests/compile_fail/not_struct_deref.stderr new file mode 100644 index 000000000..0337c2102 --- /dev/null +++ b/proc_macro/tests/compile_fail/not_struct_deref.stderr @@ -0,0 +1,10 @@ +error: zk_evm_proc_macro error: expected `struct` + --> tests/compile_fail/not_struct_deref.rs:3:1 + | +3 | / #[repr(C)] +4 | | #[derive(DerefColumns)] +5 | | enum Maybe { +6 | | Nothing, +7 | | Just(T), +8 | | } + | |_^ diff --git a/proc_macro/tests/compiletest.rs b/proc_macro/tests/compiletest.rs new file mode 100644 index 000000000..1e9d73b78 --- /dev/null +++ b/proc_macro/tests/compiletest.rs @@ -0,0 +1,14 @@ +mod compile; + +/// The `compile_fail` tests in `tests/compile_fail/` are inherently fragile, +/// as small changes in output from rustc or from the trybuild crate may cause +/// them to fail. To regenerate the `*.stderr` files, run: +/// `TRYBUILD=overwrite cargo test -p zk_evm_proc_macro -- test_compile_fail`. +/// Then, check the git diff to ensure that the new `*.stderr` files are what +/// you would expect (most importantly, make sure they actually contain errors). +#[cfg_attr(miri, ignore = "incompatible with miri")] +#[test] +fn test_compile_fail() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compile_fail/*.rs"); +} diff --git a/proof_gen/Cargo.toml b/proof_gen/Cargo.toml index 11f5fe319..b5c9169c7 100644 --- a/proof_gen/Cargo.toml +++ b/proof_gen/Cargo.toml @@ -10,7 +10,6 @@ homepage.workspace = true keywords.workspace = true [dependencies] -ethereum-types = { workspace = true } log = { workspace = true } paste = { workspace = true } plonky2 = { workspace = true } diff --git a/smt_trie/Cargo.toml b/smt_trie/Cargo.toml index fb1335350..f3008c5f7 100644 --- a/smt_trie/Cargo.toml +++ b/smt_trie/Cargo.toml @@ -14,26 +14,8 @@ keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bytes = { workspace = true } -enum-as-inner = { workspace = true } ethereum-types = { workspace = true } -hex = { workspace = true } hex-literal = { workspace = true } -keccak-hash = { workspace = true } -log = { workspace = true } -num-traits = { workspace = true } -parking_lot = { workspace = true, features = ["serde"] } plonky2 = { workspace = true } rand = { workspace = true } -rlp = { workspace = true } serde = { workspace = true, features = ["derive", "rc"] } -thiserror = { workspace = true } -uint = { workspace = true } - - -[dev-dependencies] -eth_trie = "0.4.0" -pretty_env_logger = "0.5.0" -rlp-derive = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } diff --git a/trace_decoder/Cargo.toml b/trace_decoder/Cargo.toml index d4a3629b3..0155e8126 100644 --- a/trace_decoder/Cargo.toml +++ b/trace_decoder/Cargo.toml @@ -10,18 +10,14 @@ homepage.workspace = true keywords.workspace = true [dependencies] -bytes = { workspace = true } ciborium = { workspace = true } -ciborium-io = { workspace = true } enum-as-inner = { workspace = true } enumn = { workspace = true } ethereum-types = { workspace = true } hex = { workspace = true } -hex-literal = { workspace = true } keccak-hash = { workspace = true } log = { workspace = true } rlp = { workspace = true } -rlp-derive = { workspace = true } serde = { workspace = true } serde_with = { workspace = true } thiserror = { workspace = true } diff --git a/worker.Dockerfile b/worker.Dockerfile new file mode 100644 index 000000000..865fb7bb6 --- /dev/null +++ b/worker.Dockerfile @@ -0,0 +1,40 @@ +FROM rustlang/rust:nightly-bullseye-slim as builder + +RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config + +RUN mkdir -p zero_bin +COPY Cargo.toml . +# Cleanup all workspace members and add selected crates again +RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml +RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/worker", "zero_bin\/common", "zero_bin\/ops"\, "evm_arithmetization", "mpt_trie", "proc_macro"\]#' Cargo.toml +COPY Cargo.lock . +COPY ./rust-toolchain.toml ./ + +COPY proof_gen proof_gen +COPY mpt_trie mpt_trie +COPY evm_arithmetization evm_arithmetization +COPY proc_macro proc_macro +COPY zero_bin/common zero_bin/common +COPY zero_bin/ops zero_bin/ops +COPY zero_bin/worker zero_bin/worker + +RUN \ + touch zero_bin/common/src/lib.rs && \ + touch zero_bin/ops/src/lib.rs && \ + touch zero_bin/worker/src/main.rs && \ + touch evm_arithmetization/src/lib.rs && \ + touch mpt_trie/src/lib.rs && \ + touch proc_macro/src/lib.rs + +# Disable the lld linker for now, as it's causing issues with the linkme package. +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' + +RUN cargo build --release --bin worker + +FROM debian:bullseye-slim +RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 +COPY --from=builder ./target/release/worker /usr/local/bin/worker +ENTRYPOINT ["/usr/local/bin/worker"] + diff --git a/zero_bin/.cargo/config.toml b/zero_bin/.cargo/config.toml index 708b3e6ea..e8914b049 100644 --- a/zero_bin/.cargo/config.toml +++ b/zero_bin/.cargo/config.toml @@ -1,5 +1,11 @@ [build] -rustflags = ["-C", "target-cpu=znver3"] +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +rustflags = ["-Zlinker-features=-lld"] + +[env] +# If we're running in the project workspace, then we should set the workspace env var so we read/write to/from files relative to the workspace. +CARGO_WORKSPACE_DIR = { value = "", relative = true } [target.x86_64-unknown-linux-gnu] linker = "clang" \ No newline at end of file diff --git a/zero_bin/common/src/prover_state/persistence.rs b/zero_bin/common/src/prover_state/persistence.rs index cd33b04e0..87eb7d1ae 100644 --- a/zero_bin/common/src/prover_state/persistence.rs +++ b/zero_bin/common/src/prover_state/persistence.rs @@ -17,9 +17,10 @@ use super::{ Config, RecursiveCircuitsForTableSize, SIZE, }; -const CIRCUITS_FOLDER: &str = "./circuits"; +const CIRCUITS_DIR: &str = "circuits/"; const PROVER_STATE_FILE_PREFIX: &str = "prover_state"; const VERIFIER_STATE_FILE_PREFIX: &str = "verifier_state"; +const CARGO_WORKSPACE_DIR_ENV: &str = "CARGO_WORKSPACE_DIR"; fn get_serializers() -> ( DefaultGateSerializer, @@ -72,9 +73,11 @@ pub(crate) trait DiskResource { p: &Self::PathConstrutor, r: &Self::Resource, ) -> Result<(), DiskResourceError> { + let circuits_dir = relative_circuit_dir_path(); + // Create the base folder if non-existent. - if std::fs::metadata(CIRCUITS_FOLDER).is_err() { - std::fs::create_dir(CIRCUITS_FOLDER).map_err(|_| { + if std::fs::metadata(&circuits_dir).is_err() { + std::fs::create_dir(&circuits_dir).map_err(|_| { DiskResourceError::IoError::(std::io::Error::other( "Could not create circuits folder", )) @@ -104,7 +107,7 @@ impl DiskResource for BaseProverResource { fn path(p: &Self::PathConstrutor) -> impl AsRef { format!( "{}/{}_base_{}_{}", - CIRCUITS_FOLDER, + &relative_circuit_dir_path(), PROVER_STATE_FILE_PREFIX, env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()), p.get_configuration_digest() @@ -140,7 +143,7 @@ impl DiskResource for MonolithicProverResource { fn path(p: &Self::PathConstrutor) -> impl AsRef { format!( "{}/{}_monolithic_{}_{}", - CIRCUITS_FOLDER, + &relative_circuit_dir_path(), PROVER_STATE_FILE_PREFIX, env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()), p.get_configuration_digest() @@ -175,7 +178,7 @@ impl DiskResource for RecursiveCircuitResource { fn path((circuit_type, size): &Self::PathConstrutor) -> impl AsRef { format!( "{}/{}_{}_{}_{}", - CIRCUITS_FOLDER, + &relative_circuit_dir_path(), PROVER_STATE_FILE_PREFIX, env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()), circuit_type.as_short_str(), @@ -219,7 +222,7 @@ impl DiskResource for VerifierResource { fn path(p: &Self::PathConstrutor) -> impl AsRef { format!( "{}/{}_{}_{}", - CIRCUITS_FOLDER, + &relative_circuit_dir_path(), VERIFIER_STATE_FILE_PREFIX, env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()), p.get_configuration_digest() @@ -273,3 +276,12 @@ fn prover_to_disk( Ok(()) } + +/// If we're running in the cargo workspace, then always use the `circuits` +/// directory that lives in `tools/`. Otherwise, just use `circuits` in the +/// current directory. +fn relative_circuit_dir_path() -> String { + env::var(CARGO_WORKSPACE_DIR_ENV) + .map(|p| format!("{}/{}", p, CIRCUITS_DIR)) + .unwrap_or_else(|_| CIRCUITS_DIR.to_string()) +} diff --git a/zero_bin/leader.Dockerfile b/zero_bin/leader.Dockerfile deleted file mode 100644 index 7e0eb6e8c..000000000 --- a/zero_bin/leader.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM rustlang/rust:nightly-bullseye-slim as builder - -RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config - -RUN \ - mkdir -p ops/src && touch ops/src/lib.rs && \ - mkdir -p common/src && touch common/src/lib.rs && \ - mkdir -p rpc/src && touch rpc/src/lib.rs && \ - mkdir -p prover/src && touch prover/src/lib.rs && \ - mkdir -p leader/src && echo "fn main() {println!(\"YO!\");}" > leader/src/main.rs - -COPY Cargo.toml . -RUN sed -i "2s/.*/members = [\"ops\", \"leader\", \"common\", \"rpc\", \"prover\"]/" Cargo.toml -COPY Cargo.lock . - -COPY ops/Cargo.toml ./ops/Cargo.toml -COPY common/Cargo.toml ./common/Cargo.toml -COPY rpc/Cargo.toml ./rpc/Cargo.toml -COPY prover/Cargo.toml ./prover/Cargo.toml -COPY leader/Cargo.toml ./leader/Cargo.toml - -COPY ./rust-toolchain.toml ./ - -RUN cargo build --release --bin leader - -COPY ops ./ops -COPY common ./common -COPY rpc ./rpc -COPY prover ./prover -COPY leader ./leader -RUN \ - touch ops/src/lib.rs && \ - touch common/src/lib.rs && \ - touch rpc/src/lib.rs && \ - touch prover/src/lib.rs && \ - touch leader/src/main.rs - -RUN cargo build --release --bin leader - -FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 -COPY --from=builder ./target/release/leader /usr/local/bin/leader -CMD ["leader"] diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index f136f4dbf..a2c7020f1 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -1,4 +1,4 @@ -use std::env; +use std::{env, io}; use std::{fs::File, path::PathBuf}; use anyhow::Result; @@ -9,7 +9,7 @@ use dotenvy::dotenv; use ops::register; use paladin::runtime::Runtime; use proof_gen::proof_types::GeneratedBlockProof; -use tracing::info; +use tracing::{info, warn}; use zero_bin_common::block_interval::BlockInterval; use crate::client::{client_main, ProofParams}; @@ -36,7 +36,7 @@ fn get_previous_proof(path: Option) -> Result Result<()> { - dotenv().ok(); + load_dotenvy_vars_if_present(); init::tracing(); if env::var("EVM_ARITHMETIZATION_PKG_VER").is_err() { @@ -141,3 +141,15 @@ async fn main() -> Result<()> { Ok(()) } + +/// Attempt to load in the local `.env` if present and set any environment +/// variables specified inside of it. +/// +/// To keep things simple, any IO error we will treat as the file not existing +/// and continue moving on without the `env` variables set. +fn load_dotenvy_vars_if_present() { + match dotenv() { + Ok(_) | Err(dotenvy::Error::Io(io::Error { .. })) => (), + Err(e) => warn!("Found local `.env` file but was unable to parse it! (err: {e})",), + } +} diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index 45ad4f9e5..4dc7f59c9 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -10,7 +10,6 @@ categories.workspace = true [dependencies] tokio = { workspace = true } -tracing = { workspace = true } tracing-subscriber = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } @@ -19,18 +18,15 @@ serde_json = { workspace = true } clap = { workspace = true } evm_arithmetization = { workspace = true } mpt_trie = { workspace = true } -thiserror = { workspace = true } alloy.workspace = true futures = { workspace = true } -hex = { workspace = true } -hex-literal = { workspace = true } -itertools = { workspace = true } url = { workspace = true } -__compat_primitive_types = { version = "0.12.2", package = "primitive-types" } +__compat_primitive_types = { workspace = true } tower = { workspace = true, features = ["retry"] } chrono = {workspace = true} # Local dependencies -zero_bin_common ={ workspace = true } +compat = { workspace = true } +zero_bin_common = { workspace = true } prover = { workspace = true } diff --git a/zero_bin/rpc/src/lib.rs b/zero_bin/rpc/src/lib.rs index eedc889d1..00a614521 100644 --- a/zero_bin/rpc/src/lib.rs +++ b/zero_bin/rpc/src/lib.rs @@ -8,19 +8,17 @@ use alloy::{ }; use anyhow::Context as _; use clap::ValueEnum; +use compat::Compat; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; use futures::{StreamExt as _, TryStreamExt as _}; use prover::ProverInput; use trace_decoder::types::{BlockLevelData, OtherBlockData}; use zero_bin_common::block_interval::BlockInterval; -mod compat; pub mod jerigon; pub mod native; pub mod retry; -use compat::Compat; - const PREVIOUS_HASHES_COUNT: usize = 256; /// The RPC type. diff --git a/zero_bin/rpc/src/native/state.rs b/zero_bin/rpc/src/native/state.rs index b4992848f..d69b48cb9 100644 --- a/zero_bin/rpc/src/native/state.rs +++ b/zero_bin/rpc/src/native/state.rs @@ -14,7 +14,7 @@ use trace_decoder::trace_protocol::{ SeparateTriePreImages, TrieDirect, TxnInfo, }; -use crate::compat::Compat; +use crate::Compat; /// Processes the state witness for the given block. pub async fn process_state_witness( diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 7c55e1fb7..9e1e8721a 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -24,7 +24,7 @@ use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::trace_protocol::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use super::CodeDb; -use crate::compat::Compat; +use crate::Compat; /// Processes the transactions in the given block and updates the code db. pub(super) async fn process_transactions( diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 971e3b41a..9a0700f22 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -37,7 +37,13 @@ else export MEMORY_CIRCUIT_SIZE="17..28" fi -PROOF_OUTPUT_DIR="proofs" +# Force the working directory to always be the `tools/` directory. +TOOLS_DIR=$(dirname $(realpath "$0")) + +# Set the environment variable to let the binary know that we're running in the project workspace. +export CARGO_WORKSPACE_DIR="${TOOLS_DIR}/../" + +PROOF_OUTPUT_DIR="${TOOLS_DIR}/proofs" OUT_LOG_PATH="${PROOF_OUTPUT_DIR}/b$1_$2.log" ALWAYS_WRITE_LOGS=0 # Change this to `1` if you always want logs to be written. TOT_BLOCKS=$(($2-$1+1)) @@ -57,7 +63,6 @@ RUN_VERIFICATION="${RUN_VERIFICATION:-false}" mkdir -p $PROOF_OUTPUT_DIR - if [ $IGNORE_PREVIOUS_PROOFS ]; then # Set checkpoint height to previous block number for the first block in range prev_proof_num=$(($1-1)) diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index b16dd4e5f..43f62dd59 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -11,6 +11,17 @@ # We're going to set the parallelism in line with the total cpu count num_procs=$(nproc) +# Force the working directory to always be the `tools/` directory. +TOOLS_DIR=$(dirname $(realpath "$0")) + +LEADER_OUT_PATH="${TOOLS_DIR}/leader.out" +PROOFS_JSON_PATH="${TOOLS_DIR}/proofs.json" +VERIFY_OUT_PATH="${TOOLS_DIR}/verify.out" +TEST_OUT_PATH="${TOOLS_DIR}/test.out" + +# Set the environment variable to let the binary know that we're running in the project workspace. +export CARGO_WORKSPACE_DIR="${TOOLS_DIR}/../" + # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs @@ -67,12 +78,12 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee test.out - if grep -q 'All proof witnesses have been generated successfully.' test.out; then + cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $TEST_OUT_PATH + if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" exit else - echo "Failed to create proof witnesses. See test.out for more details." + echo "Failed to create proof witnesses. See \"zk_evm/tools/test.out\" for more details." exit 1 fi fi @@ -80,14 +91,14 @@ fi cargo build --release --jobs "$num_procs" start_time=$(date +%s%N) -../../target/release/leader --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee leader.out +"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $LEADER_OUT_PATH end_time=$(date +%s%N) -tail -n 1 leader.out > proofs.json +tail -n 1 $LEADER_OUT_PATH > $PROOFS_JSON_PATH -../../target/release/verifier -f proofs.json | tee verify.out +"${TOOLS_DIR}/../../target/release/verifier" -f $PROOFS_JSON_PATH | tee $VERIFY_OUT_PATH -if grep -q 'All proofs verified successfully!' verify.out; then +if grep -q 'All proofs verified successfully!' $VERIFY_OUT_PATH; then duration_ns=$((end_time - start_time)) duration_sec=$(echo "$duration_ns / 1000000000" | bc -l) echo "Success!" diff --git a/zero_bin/verifier/Cargo.toml b/zero_bin/verifier/Cargo.toml index cbfb0123c..40dd297b7 100644 --- a/zero_bin/verifier/Cargo.toml +++ b/zero_bin/verifier/Cargo.toml @@ -10,7 +10,6 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } dotenvy = { workspace = true } anyhow = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } serde_path_to_error = { workspace = true } proof_gen = { workspace = true } diff --git a/zero_bin/worker.Dockerfile b/zero_bin/worker.Dockerfile deleted file mode 100644 index 39036aa2c..000000000 --- a/zero_bin/worker.Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM rustlang/rust:nightly-bullseye-slim as builder - -RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev - -RUN \ - mkdir -p common/src && touch common/src/lib.rs && \ - mkdir -p ops/src && touch ops/src/lib.rs && \ - mkdir -p worker/src && echo "fn main() {println!(\"YO!\");}" > worker/src/main.rs - -COPY Cargo.toml . -RUN sed -i "2s/.*/members = [\"common\", \"ops\", \"worker\"]/" Cargo.toml -COPY Cargo.lock . - -COPY common/Cargo.toml ./common/Cargo.toml -COPY ops/Cargo.toml ./ops/Cargo.toml -COPY worker/Cargo.toml ./worker/Cargo.toml - -COPY ./rust-toolchain.toml ./ - -RUN cargo build --release --bin worker - -COPY common ./common -COPY ops ./ops -COPY worker ./worker -RUN \ - touch common/src/lib.rs && \ - touch ops/src/lib.rs && \ - touch worker/src/main.rs - -RUN cargo build --release --bin worker - -FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 -COPY --from=builder ./target/release/worker /usr/local/bin/worker -CMD ["worker"] diff --git a/zero_bin/worker/Cargo.toml b/zero_bin/worker/Cargo.toml index 90f13bd14..16d55ef29 100644 --- a/zero_bin/worker/Cargo.toml +++ b/zero_bin/worker/Cargo.toml @@ -12,7 +12,6 @@ categories.workspace = true paladin-core = { workspace = true } anyhow = { workspace = true } dotenvy = { workspace = true } -tracing = { workspace = true } tracing-subscriber = { workspace = true } clap = { workspace = true } tokio = { workspace = true } @@ -22,4 +21,4 @@ ops = { workspace = true } zero_bin_common = { workspace = true } [target.'cfg(not(target_env = "msvc"))'.dependencies] -jemallocator = "0.5.0" +jemallocator = "0.5.4"