WIP Pipelines and dockerfiles

.github/workflows/deploy.yml

@@ -0,0 +1,122 @@
+name: Build & Deploy
+
+on:
+  push:
+    branches:
+      - develop
+      - pipelines-and-dockerfiles
+
+env:
+  DEFAULT_TAGS: |
+    type=ref,event=branch
+    type=ref,event=pr
+    type=semver,pattern={{version}}
+    type=semver,pattern={{major}}.{{minor}}
+    type=sha,format=long,event=branch
+  IMAGE_BASE_NAME: us.gcr.io/${{ secrets.GCP_PROJECT_ID }}
+
+jobs:
+  build:
+    name: Build docker images
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Get docker image meta (leader)
+        id: leader-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_BASE_NAME }}/zero-bin-leader
+          tags: ${{ env.DEFAULT_TAGS }}
+
+      - name: Get docker image meta (worker)
+        id: worker-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_BASE_NAME }}/zero-bin-worker
+          tags: ${{ env.DEFAULT_TAGS }}
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_ID }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+          token_format: 'access_token'
+          access_token_lifetime: '900s'
+
+      - name: Login to GCR
+        uses: docker/login-action@v3
+        with:
+          registry: us.gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - name: Build and push worker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: worker.Dockerfile
+          push: true
+          tags: ${{ steps.worker-meta.outputs.tags }}
+          labels: ${{ steps.worker-meta.outputs.labels }}
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+
+      - name: Build and push leader
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: coordinator.Dockerfile
+          push: true
+          tags: ${{ steps.leader-meta.outputs.tags }}
+          labels: ${{ steps.leader-meta.outputs.labels }}
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+
+  deploy:
+    name: Deploy to GKE
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_ID }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: Get GKE credentials
+        id: 'get-credentials'
+        uses: 'google-github-actions/get-gke-credentials@v2'
+        with:
+          cluster_name: 'immutable-prod'
+          location: 'us-central1'
+
+      - name: Install helm
+        uses: azure/[email protected]
+        with:
+          version: 3.14.3
+        id: install
+
+      - name: Deploy to GKE
+        id: deploy
+        run: |-
+          pwd && ls -lh
+          cd ./deploy/helm
+          helm upgrade zero-bin ./zero-bin -f ./zero-bin/values.yaml --set hull.config.specific.version=sha-${GITHUB_SHA} -n zkevm --install

Cargo.toml

@@ -1,16 +1,5 @@
 [workspace]
-members = ["mpt_trie",
-    "proof_gen",
-    "trace_decoder",
-    "evm_arithmetization",
-    "zero_bin/leader",
-    "zero_bin/worker",
-    "zero_bin/common",
-    "zero_bin/ops",
-    "zero_bin/verifier",
-    "zero_bin/rpc",
-    "zero_bin/prover",
-    "zero_bin/coordinator"]
+members = [ "mpt_trie", "proof_gen", "trace_decoder", "evm_arithmetization", "zero_bin/leader", "zero_bin/worker", "zero_bin/common", "zero_bin/ops", "zero_bin/verifier", "zero_bin/rpc", "zero_bin/prover", "zero_bin/coordinator" ]
 resolver = "2"
 
 [workspace.package]
@@ -22,20 +11,7 @@ keywords = ["cryptography", "STARK", "plonky2", "ethereum", "zk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [workspace.dependencies]
-alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-features = false, features = [
-    "consensus",
-    "reqwest",
-    "json-rpc",
-    "rlp",
-    "rpc",
-    "rpc-client",
-    "rpc-types-eth",
-    "rpc-types-trace",
-    "providers",
-    "transports",
-    "transport-http",
-    "rpc-types-debug"
-] }
+alloy = { git = "https://github.com/alloy-rs/alloy", tag = 'v0.1.1', default-features = false, features = [ "consensus", "reqwest", "json-rpc", "rlp", "rpc", "rpc-client", "rpc-types-eth", "rpc-types-trace", "providers", "transports", "transport-http", "rpc-types-debug" ] }
 anyhow = "1.0.40"
 async-stream = "0.3.5"
 axum = "0.7.4"
@@ -115,4 +91,4 @@ google-cloud-storage = "0.16.0"
 google-cloud-googleapis = "0.12.0"
 google-cloud-auth = "0.13.2"
 
-chrono = "0.4.38"
+chrono = "0.4.38"

coordinator.Dockerfile

@@ -0,0 +1,48 @@
+FROM rustlang/rust:nightly-bullseye-slim as builder
+
+RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config
+
+RUN \
+    mkdir -p ops/src     && touch ops/src/lib.rs && \
+    mkdir -p common/src  && touch common/src/lib.rs && \
+    mkdir -p rpc/src     && touch rpc/src/lib.rs && \
+    mkdir -p prover/src  && touch prover/src/lib.rs && \
+    mkdir -p leader/src && touch leader/src/lib.rs && \
+    mkdir -p coordinator/src  && echo "fn main() {println!(\"coordinator main\");}" > coordinator/src/main.rs
+
+COPY Cargo.toml .
+RUN sed -i "2s/.*/members = [\"ops\", \"leader\", \"common\", \"rpc\", \"prover\", \"coordinator\"]/" Cargo.toml
+COPY Cargo.lock .
+
+COPY zero_bin/ops/Cargo.toml ./ops/Cargo.toml
+COPY zero_bin/common/Cargo.toml ./common/Cargo.toml
+COPY zero_bin/rpc/Cargo.toml ./rpc/Cargo.toml
+COPY zero_bin/prover/Cargo.toml ./prover/Cargo.toml
+COPY zero_bin/leader/Cargo.toml ./leader/Cargo.toml
+COPY zero_bin/coordinator/Cargo.toml ./coordinator/Cargo.toml
+
+COPY ./rust-toolchain.toml ./
+
+RUN cargo build --verbose --release --bin coordinator
+
+COPY zero_bin/coordinator ./coordinator
+COPY zero_bin/ops ./ops
+COPY zero_bin/common ./common
+COPY zero_bin/rpc ./rpc
+COPY zero_bin/prover ./prover
+COPY zero_bin/leader ./leader
+
+RUN \
+    touch ops/src/lib.rs && \
+    touch common/src/lib.rs && \
+    touch rpc/src/lib.rs && \
+    touch prover/src/lib.rs && \
+    touch leader/src/main.rs && \
+    touch coordinator/src/main.rs
+
+RUN cargo build --verbose --release --bin coordinator
+
+FROM debian:bullseye-slim
+RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 make libssl-dev
+COPY --from=builder ./target/release/coordinator /usr/local/bin/coordinator
+CMD ["coordinator"]

zero_bin/worker.Dockerfile → worker.Dockerfile

@@ -1,6 +1,6 @@
 FROM rustlang/rust:nightly-bullseye-slim as builder
 
-RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev
+RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config
 
 RUN \
   mkdir -p common/src  && touch common/src/lib.rs && \
@@ -11,17 +11,17 @@ COPY Cargo.toml .
 RUN sed -i "2s/.*/members = [\"common\", \"ops\", \"worker\"]/" Cargo.toml
 COPY Cargo.lock .
 
-COPY common/Cargo.toml ./common/Cargo.toml
-COPY ops/Cargo.toml ./ops/Cargo.toml
-COPY worker/Cargo.toml ./worker/Cargo.toml
+COPY zero_bin/common/Cargo.toml ./common/Cargo.toml
+COPY zero_bin/ops/Cargo.toml ./ops/Cargo.toml
+COPY zero_bin/worker/Cargo.toml ./worker/Cargo.toml
 
 COPY ./rust-toolchain.toml ./
 
 RUN cargo build --release --bin worker
 
-COPY common ./common
-COPY ops ./ops
-COPY worker ./worker
+COPY zero_bin/common ./common
+COPY zero_bin/ops ./ops
+COPY zero_bin/worker ./worker
 RUN \
   touch common/src/lib.rs && \
   touch ops/src/lib.rs && \