feat: parallelise at hole level #78
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| testbed: | |
| strategy: | |
| matrix: | |
| repo: [simple, mmaitre314/picklescan, huggingface/huggingface_hub, tiangolo/fastapi, encode/starlette, lancedb/lancedb, lancedb/lance, tkaitchuck/constrandom, jaemk/cached, smol-rs/async-executor, gcanti/io-ts, colinhacks/zod, helix-editor/helix] | |
| runs-on: [self-hosted, intel-cpu, 8-cpu, ci] | |
| container: | |
| image: ubuntu:22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| apt update | |
| DEBIAN_FRONTEND=noninteractive apt install -y pkg-config protobuf-compiler libssl-dev curl build-essential git-all gfortran | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| rustflags: '' | |
| toolchain: nightly | |
| - name: Install Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install node 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Install yarn | |
| run: | | |
| npm i -g yarn | |
| - name: Set up cargo cache | |
| uses: actions/cache@v3 | |
| continue-on-error: false | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| - name: Build project | |
| run: cargo build -r | |
| - name: Run testbed | |
| run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml -f ${{ matrix.repo }}' | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| env: | |
| API_TOKEN: ${{ secrets.API_TOKEN }} | |
| - name: Run testbed | |
| run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo }}' | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| API_TOKEN: ${{ secrets.API_TOKEN }} | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: results-${{ matrix.repo }} | |
| path: ./results.json | |
| comment_results: | |
| runs-on: [self-hosted, intel-cpu, 8-cpu, ci] | |
| container: | |
| image: ubuntu:22.04 | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| apt update | |
| apt install -y jq | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-simple | |
| path: results-simple.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-mmaitre314/picklescan | |
| path: results-picklescan.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-huggingface/huggingface_hub | |
| path: results-huggingface_hub.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-tiangolo/fastapi | |
| path: results-fastapi.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-encode/starlette | |
| path: results-starlette.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-lancedb/lancedb | |
| path: results-lancedb.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-lancedb/lancedb | |
| path: results-lancedb.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-lancedb/lance | |
| path: results-lance.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-tkaitchuck/constrandom | |
| path: results-constrandom.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-jaemk/cached | |
| path: results-cached.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-smol-rs/async-executor | |
| path: results-async-executor.json | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-gcanti/io-ts | |
| path: results-io-ts | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-colinhacks/zod | |
| path: results-zod | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: results-helix-editor/helix | |
| path: results-helix.json | |
| - name: output to markdown | |
| run: | | |
| cat > results.md <<EOF | |
| | Repository name | Source type | Average hole completion time (s) | Pass percentage | | |
| | :-------------- | :---------- | -------------------------------: | --------------: | | |
| EOF | |
| cat results-*.json | jq -r '"| \(.[0].repo_name) | \(.[0].source_type) | \(.[0].avg_hole_completion_time_ms) | \(.[0].pass_percentage)% |"' >> results.md | |
| cat >> results.md <<EOF | |
| **Note:** The "hole completion time" represents the full process of: | |
| - copying files from the setup cache directory | |
| - replacing the code from the file with a completion from the model | |
| - building the project | |
| - running the tests | |
| EOF | |
| - name: Find Comment | |
| uses: peter-evans/find-comment@v2 | |
| id: fc | |
| if: github.event_name == 'pull_request' | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: '| Repository name | Source type | Average hole completion time (s) | Pass percentage |' | |
| - name: Create or update comment | |
| if: github.event_name == 'pull_request' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: results.md | |
| edit-mode: replace | |