Merge branch 'main' into bg/algorithm-metrics

worldcoin · Jan 9, 2025 · e65366c · e65366c
2 parents fcea2b3 + 2fb8f25
commit e65366c
Show file tree

Hide file tree

Showing 39 changed files with 1,019 additions and 647 deletions.
diff --git a/.github/workflows/temp-branch-build-and-push.yaml b/.github/workflows/temp-branch-build-and-push.yaml
@@ -3,7 +3,7 @@ name: Branch - Build and push docker image
 on:
   push:
     branches:
-      - "ertugrul/change-to-binary"
+      - "ps/potential-phantom-match"
 
 concurrency:
   group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -36,6 +36,7 @@ hawk-pack = { git = "https://github.com/Inversed-Tech/hawk-pack.git", rev = "ba9
 hex = "0.4.3"
 itertools = "0.13"
 num-traits = "0.2"
+memmap2 = "0.9.5"
 serde = { version = "1.0", features = ["derive"] }
 serde-big-array = "0.5.1"
 serde_json = "1"

diff --git a/Dockerfile.nocuda b/Dockerfile.nocuda
@@ -29,12 +29,12 @@ FROM --platform=linux/amd64 build-image as build-app
 WORKDIR /src/gpu-iris-mpc
 COPY . .
 
-RUN cargo build --release --target x86_64-unknown-linux-gnu --bin seed-v1-dbs --bin upgrade-server --bin upgrade-client --bin upgrade-checker --bin reshare-server
+RUN cargo build --release --target x86_64-unknown-linux-gnu --bin seed-v1-dbs --bin upgrade-server --bin upgrade-client --bin upgrade-checker --bin reshare-server --bin key-manager
 
 FROM --platform=linux/amd64 ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y ca-certificates awscli
 COPY certs /usr/local/share/ca-certificates/
 RUN update-ca-certificates
 
@@ -43,6 +43,7 @@ COPY --from=build-app /src/gpu-iris-mpc/target/x86_64-unknown-linux-gnu/release/
 COPY --from=build-app /src/gpu-iris-mpc/target/x86_64-unknown-linux-gnu/release/upgrade-client /bin/upgrade-client
 COPY --from=build-app /src/gpu-iris-mpc/target/x86_64-unknown-linux-gnu/release/upgrade-checker /bin/upgrade-checker
 COPY --from=build-app /src/gpu-iris-mpc/target/x86_64-unknown-linux-gnu/release/reshare-server /bin/reshare-server
+COPY --from=build-app /src/gpu-iris-mpc/target/x86_64-unknown-linux-gnu/release/key-manager /bin/key-manager
 
 USER 65534
 ENTRYPOINT ["/bin/upgrade-server"]
diff --git a/deploy/e2e/iris-mpc-0.yaml.tpl b/deploy/e2e/iris-mpc-0.yaml.tpl
@@ -1,8 +1,8 @@
 iris-mpc-0:
-  fullNameOverride: "iris-mpc-0"
-  image: "ghcr.io/worldcoin/iris-mpc:v0.12.2"
+  fullnameOverride: "iris-mpc-0"
+  image: "ghcr.io/worldcoin/iris-mpc:$IRIS_MPC_IMAGE_TAG"
 
-  environment: e2e
+  environment: $ENV
   replicaCount: 1
 
   strategy:
@@ -35,34 +35,34 @@ iris-mpc-0:
       path: /ready
       port: health
 
+  podSecurityContext:
+    runAsNonRoot: false
+    seccompProfile:
+      type: RuntimeDefault
+
   resources:
     limits:
       cpu: 31
-      memory: 600Gi
+      memory: 60Gi
       nvidia.com/gpu: 1
-      vpc.amazonaws.com/efa: 1
+
     requests:
       cpu: 30
       memory: 55Gi
       nvidia.com/gpu: 1
-      vpc.amazonaws.com/efa: 1
 
   imagePullSecrets:
     - name: github-secret
 
   nodeSelector:
     kubernetes.io/arch: amd64
 
-  hostNetwork: true
-
-  podSecurityContext:
-    runAsUser: 65534
-    runAsGroup: 65534
+  hostNetwork: false
 
   tolerations:
-    - key: "dedicated"
+    - key: "gpuGroup"
       operator: "Equal"
-      value: "gpuGroup"
+      value: "dedicated"
       effect: "NoSchedule"
 
   keelPolling:
@@ -90,20 +90,29 @@ iris-mpc-0:
     - name: RUST_LOG
       value: "info"
 
+    - name: AWS_REGION
+      value: "$AWS_REGION"
+
+    - name: AWS_ENDPOINT_URL
+      value: "http://localstack:4566"
+
     - name: RUST_BACKTRACE
       value: "full"
 
     - name: NCCL_SOCKET_IFNAME
       value: "eth0"
 
     - name: NCCL_COMM_ID
-      value: "iris-mpc-node.1.e2e.smpcv2.worldcoin.dev:4000"
+      value: "iris-mpc-0.svc.cluster.local:4000"
 
     - name: SMPC__ENVIRONMENT
-      value: "e2e"
+      value: "$ENV"
+
+    - name: SMPC__AWS__REGION
+      value: "$AWS_REGION"
 
     - name: SMPC__SERVICE__SERVICE_NAME
-      value: "smpcv2-server-e2e"
+      value: "smpcv2-server-$ENV"
 
     - name: SMPC__DATABASE__URL
       valueFrom:
@@ -120,9 +129,6 @@ iris-mpc-0:
     - name: SMPC__DATABASE__LOAD_PARALLELISM
       value: "8"
 
-    - name: SMPC__AWS__REGION
-      value: "eu-north-1"
-
     - name: SMPC__REQUESTS_QUEUE_URL
       value: "arn:aws:sns:eu-central-1:000000000000:iris-mpc-input"
 
@@ -136,17 +142,13 @@ iris-mpc-0:
       value: "/data/"
 
     - name: SMPC__KMS_KEY_ARNS
-      value: [
-        "arn:aws:kms:eu-north-1:000000000000:key/00000000-0000-0000-0000-000000000000",
-        "arn:aws:kms:eu-north-1:000000000000:key/00000000-0000-0000-0000-000000000001",
-        "arn:aws:kms:eu-north-1:000000000000:key/00000000-0000-0000-0000-000000000002"
-      ]
+      value: '["arn:aws:kms:$AWS_REGION:000000000000:key/00000000-0000-0000-0000-000000000000","arn:aws:kms:$AWS_REGION:000000000000:key/00000000-0000-0000-0000-000000000001","arn:aws:kms:$AWS_REGION:000000000000:key/00000000-0000-0000-0000-000000000002"]'
 
     - name: SMPC__PARTY_ID
       value: "0"
 
     - name: SMPC__PUBLIC_KEY_BASE_URL
-      value: "https://pki-smpcv2-stage.worldcoin.org"
+      value: "http://wf-$ENV-public-keys.s3.localhost.localstack.cloud:4566"
 
     - name: SMPC__ENABLE_S3_IMPORTER
       value: "false"
@@ -181,59 +183,39 @@ iris-mpc-0:
       value: "256"
 
     - name: SMPC__SERVICE__METRICS__PREFIX
-      value: "smpcv2-e2e-0"
+      value: "smpcv2-$ENV-0"
 
     - name: SMPC__RETURN_PARTIAL_RESULTS
       value: "true"
 
     - name: SMPC__NODE_HOSTNAMES
-      value: '["iris-mpc-node.1.e2e.smpcv2.worldcoin.dev","iris-mpc-node.2.e2e.smpcv2.worldcoin.dev","iris-mpc-node.3.e2e.smpcv2.worldcoin.dev"]'
+      value: '["iris-mpc-0.svc.cluster.local","iris-mpc-1.svc.cluster.local","iris-mpc-2.svc.cluster.local"]'
 
     - name: SMPC__IMAGE_NAME
-      value: $(IMAGE_NAME)
+      value: "ghcr.io/worldcoin/iris-mpc:$IRIS_MPC_IMAGE_TAG"
 
   initContainer:
     enabled: true
-    image: "amazon/aws-cli:2.17.62"
-    name: "iris-mpc-copy-cuda-libs"
+    image: "ghcr.io/worldcoin/iris-mpc:2694d8cbb37c278ed84951ef9aac3af47b21f146" # no-cuda image
+    name: "iris-mpc-0-copy-cuda-libs"
     env:
+      - name: AWS_REGION
+        value: "$AWS_REGION"
       - name: PARTY_ID
         value: "1"
       - name: MY_NODE_IP
         valueFrom:
           fieldRef:
             fieldPath: status.hostIP
     configMap:
+      name: "iris-mpc-0-init"
       init.sh: |
         #!/usr/bin/env bash
-
-        # Set up environment variables
-        HOSTED_ZONE_ID=$(aws route53 list-hosted-zones-by-name --dns-name "$PARTY_ID".e2e.smpcv2.worldcoin.dev --query "HostedZones[].Id" --output text)
-
-        # Generate the JSON content in memory
-        BATCH_JSON=$(cat <<EOF
-        {
-          "Comment": "Upsert the A record for iris-mpc NCCL_COMM_ID",
-          "Changes": [
-            {
-              "Action": "UPSERT",
-              "ResourceRecordSet": {
-                "Name": "iris-mpc-node.$PARTY_ID.e2e.smpcv2.worldcoin.dev",
-                "TTL": 5,
-                "Type": "A",
-                "ResourceRecords": [{
-                  "Value": "$MY_NODE_IP"
-                }]
-              }
-            }
-          ]
-        }
-        EOF
-        )
-
-        # Execute AWS CLI command with the generated JSON
-        aws route53 change-resource-record-sets --hosted-zone-id "$HOSTED_ZONE_ID" --change-batch "$BATCH_JSON"
-
+        set -e
+
         cd /libs
+
         aws s3 cp s3://wf-smpcv2-stage-libs/libcublas.so.12.2.5.6 .
         aws s3 cp s3://wf-smpcv2-stage-libs/libcublasLt.so.12.2.5.6 .
+
+        key-manager --node-id 0 --env $ENV --endpoint-url "http://localstack:4566" rotate --public-key-bucket-name wf-$ENV-stage-public-keys --region $AWS_REGION