From 234e99b535d70e867b9054829210de47858dc9a6 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 9 Oct 2024 16:34:48 +0300 Subject: [PATCH 1/8] testing a pr push-to-registry --- .github/workflows/benchmarking.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/benchmarking.yml diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml new file mode 100644 index 0000000000..be184891c7 --- /dev/null +++ b/.github/workflows/benchmarking.yml @@ -0,0 +1,29 @@ +on: + pull_request: + +defaults: + run: + working-directory: packages/sync-service + +env: + PROJECT_ID: vaxine + REGISTRY: europe-docker.pkg.dev/vaxine/electric + +jobs: + build: + name: Build and test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCLOUD_REGISTRY_JSON_KEY }} + - uses: google-github-actions/setup-gcloud@v1 + - name: Use gcloud CLI + run: "gcloud info" + - name: Docker auth + run: gcloud auth configure-docker europe-docker.pkg.dev --quiet + - name: Pull latest base image + run: docker pull electricsql/electric:canary + - name: Build image + run: docker build --push --cache-from=electricsql/electric:canary --tag ${{ env.REGISTRY }}/electric:${{ github.event.number }} . From 2d5b0ca0d355ae50790ea7f223bf7b5746c9a082 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 9 Oct 2024 16:52:02 +0300 Subject: [PATCH 2/8] Updated the building process for better caching --- .github/workflows/benchmarking.yml | 35 ++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index be184891c7..68db1442c2 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -23,7 +23,34 @@ jobs: run: "gcloud info" - name: Docker auth run: gcloud auth configure-docker europe-docker.pkg.dev --quiet - - name: Pull latest base image - run: docker pull electricsql/electric:canary - - name: Build image - run: docker build --push --cache-from=electricsql/electric:canary --tag ${{ env.REGISTRY }}/electric:${{ github.event.number }} . + - name: Pull latest base images + run: | + docker pull ${{ env.REGISTRY }}/electric:canary-builder || true + docker pull ${{ env.REGISTRY }}/electric:canary-runner-base || true + docker pull ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder || true + docker pull ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base || true + - name: Build base images + run: > + docker build + --push + --cache-from=${{ env.REGISTRY }}/electric:canary-builder + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder + --target builder + . + + docker build + --push + --cache-from=${{ env.REGISTRY }}/electric:canary-runner-base + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base + --target runner_setup + . + - name: Build actual image + run: > + docker build + --push + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }} + . From d467ac026d6c7bc37b486294af05384a46254f4f Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 9 Oct 2024 18:04:34 +0300 Subject: [PATCH 3/8] figuring out CI --- .github/workflows/benchmarking.yml | 53 ++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 68db1442c2..a917125a42 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -31,26 +31,45 @@ jobs: docker pull ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base || true - name: Build base images run: > - docker build - --push - --cache-from=${{ env.REGISTRY }}/electric:canary-builder - --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder - --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder - --target builder + docker build \ + --push \ + --cache-from ${{ env.REGISTRY }}/electric:canary-builder \ + --cache-from ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder \ + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder \ + --target builder \ . - docker build - --push - --cache-from=${{ env.REGISTRY }}/electric:canary-runner-base - --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base - --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base - --target runner_setup + docker build \ + --push \ + --cache-from ${{ env.REGISTRY }}/electric:canary-runner-base \ + --cache-from ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base \ + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base \ + --target runner_setup \ . - name: Build actual image run: > - docker build - --push - --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder - --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base - --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }} + docker build \ + --push \ + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder \ + --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base \ + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }} \ . + # - name: Trigger a benchmark start + # run: | + # curl -X POST 'http://localhost:4000/api/benchmarks/write_fanout/runs' \ + # -H 'Content-Type: application/json' \ + # -d '{ + # "benchmark_run": { + # "spec_values": { + # "electric_image": ["electricsql/electric:'"$ELECTRIC_VERSION"'"], + # "postgres_image": ["postgres:15-alpine"], + # "row_count": [500], + # "concurrent": [4,36,68,100,132,164,200], + # "tx_row_count": [10,410,810,1210,1610,1910] + # }, + # "machine_request": { + # "vcpu": 4, + # "mem_gb": 8 + # } + # } + # }' \ No newline at end of file From b121c2aad860896bcb0a1dc35d45dc0ccdd81357 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 9 Oct 2024 18:31:32 +0300 Subject: [PATCH 4/8] Try triggering a small benchmark --- .github/workflows/benchmarking.yml | 39 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index a917125a42..987d5c5f31 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -54,22 +54,23 @@ jobs: --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base \ --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }} \ . - # - name: Trigger a benchmark start - # run: | - # curl -X POST 'http://localhost:4000/api/benchmarks/write_fanout/runs' \ - # -H 'Content-Type: application/json' \ - # -d '{ - # "benchmark_run": { - # "spec_values": { - # "electric_image": ["electricsql/electric:'"$ELECTRIC_VERSION"'"], - # "postgres_image": ["postgres:15-alpine"], - # "row_count": [500], - # "concurrent": [4,36,68,100,132,164,200], - # "tx_row_count": [10,410,810,1210,1610,1910] - # }, - # "machine_request": { - # "vcpu": 4, - # "mem_gb": 8 - # } - # } - # }' \ No newline at end of file + - name: Trigger a benchmark start + run: | + curl -X POST 'http://benchmarking.electric-sql.com:4000/api/benchmarks/write_fanout/runs' \ + -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ + -H 'Content-Type: application/json' \ + -d '{ + "benchmark_run": { + "spec_values": { + "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }}"], + "postgres_image": ["postgres:15-alpine"], + "row_count": [500], + "concurrent": [4,36,68,100,132,164,200], + "tx_row_count": [50] + }, + "machine_request": { + "vcpu": 4, + "mem_gb": 8 + } + } + }' | jq From ffe0121586170be7f4f0911904e5b166874117a5 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Mon, 21 Oct 2024 12:45:55 +0300 Subject: [PATCH 5/8] Added actual benchmark running --- .github/workflows/benchmarking.yml | 32 ++++++++++--- .github/workflows/leave_benchmark_comment.yml | 47 +++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/leave_benchmark_comment.yml diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 987d5c5f31..7671f696b2 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -1,5 +1,6 @@ on: - pull_request: + issue_comment: + types: [created] defaults: run: @@ -23,6 +24,8 @@ jobs: run: "gcloud info" - name: Docker auth run: gcloud auth configure-docker europe-docker.pkg.dev --quiet + - name: Set outputs + run: echo "SHORT_SHA=$(git rev-parse --short=5 HEAD)" >> $GITHUB_ENV - name: Pull latest base images run: | docker pull ${{ env.REGISTRY }}/electric:canary-builder || true @@ -52,25 +55,40 @@ jobs: --push \ --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-builder \ --cache-from=${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-runner-base \ - --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }} \ + --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ env.SHORT_SHA }} \ . - - name: Trigger a benchmark start + - name: Write fanout benchmark run: | - curl -X POST 'http://benchmarking.electric-sql.com:4000/api/benchmarks/write_fanout/runs' \ + curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/write_fanout/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ + --fail \ -d '{ "benchmark_run": { "spec_values": { - "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ github.sha }}"], + "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:15-alpine"], "row_count": [500], - "concurrent": [4,36,68,100,132,164,200], + "concurrent": [5, 105, 205, 305, 405, 505, 605, 705, 805, 905, 1005], "tx_row_count": [50] }, "machine_request": { "vcpu": 4, "mem_gb": 8 + }, + "metadata": { + "pr": ${{ github.event.number }}, + "short_version": "pr-${{ github.event.number }}-${{ env.SHORT_SHA }}" + "callback": { + "method": "POST", + "headers": [ + ["Accept","application/vnd.github+json"], + ["Authorization","Bearer ${{ secrets.GITHUB_ACTION_EXECUTE_TOKEN }}"], + ["X-GitHub-Api-Version","2022-11-28"]] + ], + "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", + "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" + } } } - }' | jq + }' diff --git a/.github/workflows/leave_benchmark_comment.yml b/.github/workflows/leave_benchmark_comment.yml new file mode 100644 index 0000000000..b5c944a926 --- /dev/null +++ b/.github/workflows/leave_benchmark_comment.yml @@ -0,0 +1,47 @@ +name: Benchmark comments + +on: + workflow_dispatch: + inputs: + pr: + type: number + required: true + description: "PR for which the benchmark ran" + benchmark_info: + type: string + required: true + description: JSON-serialized info about completed benchmark allowing the job to pull the info + original_commit: + type: string + required: true + description: sha of the commit that these benchmarks are valid for + +jobs: + leave_comment: + permissions: + pull-requests: write + contents: read + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/github-script@v7 + id: get-pr + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const issue_number = context.payload.inputs.pr + const info = JSON.parse(context.payload.inputs.benchmark_info) + + const comments = await github.rest.issues.listComments({owner, repo, issue_number}) + const prefix = `Benchmark results, triggered for ${context.payload.inputs.original_commit}` + const this_graph = `\n\n- ${info.readable_name} ${info.status}\n\n[${info.readable_name} results](https://benchmarking.electric-sql.com/public_api/graphs/${info.name}/${info.id}/against_latest.svg)` + const existing = comments.data.find(x => x.body?.startsWith(prefix)) + + if (existing) { + await github.rest.issues.updateComment({owner, repo, comment_id: existing.id, body: existing.body + this_graph}) + } else { + await github.rest.issues.createComment({owner, repo, issue_number, body: prefix + this_graph}) + } From 5e425c82dee0fa9dd8bd2e7cd740281a9ef492cf Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Mon, 21 Oct 2024 12:50:47 +0300 Subject: [PATCH 6/8] Fixed the checkout --- .github/workflows/benchmarking.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 7671f696b2..7e41a6da4d 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -12,10 +12,30 @@ env: jobs: build: - name: Build and test + permissions: + pull-requests: write + contents: read + name: Build and test on PR comment + if: ${{ github.event.issue.pull_request && (startsWith(github.event.comment.body, "benchmark this") || startsWith(github.event.comment.body, "Benchmark this"))}} runs-on: ubuntu-latest steps: + - uses: actions/github-script@v7 + id: get-pr + with: + script: | + const request = { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + } + core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`) + + const result = await github.rest.pulls.get(request) + core.info(`GOT ${JSON.stringify(result)}`) + return result.data - uses: actions/checkout@v4 + with: + ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }} - uses: google-github-actions/auth@v1 with: credentials_json: ${{ secrets.GCLOUD_REGISTRY_JSON_KEY }} From eb6d4c17563b3a14710ce11d3e521d22fc89cad5 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Mon, 21 Oct 2024 12:53:27 +0300 Subject: [PATCH 7/8] Fixed naming --- .github/workflows/benchmarking.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 7e41a6da4d..007cfdcbcf 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -1,3 +1,5 @@ +name: Benchmark a PR + on: issue_comment: types: [created] From 4009c0e6af2ed70960322e0a28ecd7961519d054 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Mon, 21 Oct 2024 12:59:34 +0300 Subject: [PATCH 8/8] updated used secret --- .github/workflows/benchmarking.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 007cfdcbcf..cbd4ff64e4 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -105,7 +105,7 @@ jobs: "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], - ["Authorization","Bearer ${{ secrets.GITHUB_ACTION_EXECUTE_TOKEN }}"], + ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"]] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches",