Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set up PR-time workflow to check permission issue #153

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Set up PR-time workflow to check permission issue #153

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
311 changes: 311 additions & 0 deletions .github/workflows/continuous_tests_nonblocking.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,311 @@
name: PR-time tests

on:
pull_request:
branches: [ main ]

env:
DISTRO: ubuntu
REGISTRY: ghcr.io
FBPCF_LOCAL_IMAGE_NAME: ${{ github.event.repository.name }}/ubuntu
REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/ubuntu
COORDINATOR_IMAGE_DEV: ghcr.io/facebookresearch/fbpcs/coordinator:dev
LOCAL_IMAGE_NAME: fbpcs/onedocker
TEST_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/test/ubuntu
PL_CONTAINER_NAME: e2e_pl_container
PA_CONTAINER_NAME: e2e_pa_container
VERSION_TAG: latest-build

permissions:
contents: write
packages: write

jobs:
build_docker:
runs-on: [self-hosted, e2e_test_runner]

steps:
- uses: actions/checkout@v2

- name: Remove unused images
run: |
docker image prune -af

- name: Build fbpcf docker image
run: |
./build-docker.sh -u

- name: Sanity check fbpcf library
timeout-minutes: 3
run: |
./run-millionaire-sample.sh -u

- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Clone latest stable fbpcs
run: |
git clone https://github.com/facebookresearch/fbpcs
cd fbpcs
git reset --hard $(curl \
--header 'content-type: application/json' \
"https://api.github.com/repos/facebookresearch/fbpcs/actions/workflows/12965519/runs?per_page=1&status=success" | jq \
".workflow_runs[0] .head_sha" | tr -d '"')

- name: Build fbpcs image (this uses the locally built fbpcf image as a dependency)
run: |
cd fbpcs
./build-docker.sh onedocker -t ${{ env.VERSION_TAG }}

- name: Tag docker image
run: |
docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ env.VERSION_TAG }} ${{ env.TEST_REGISTRY_IMAGE_NAME }}:${{ github.sha }}
docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ env.VERSION_TAG }} ${{ env.TEST_REGISTRY_IMAGE_NAME }}:${{ env.VERSION_TAG }}
docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ env.VERSION_TAG }} ${{ env.TEST_REGISTRY_IMAGE_NAME }}:test_image

- name: Push image to test registry
run: |
docker push --all-tags ${{ env.TEST_REGISTRY_IMAGE_NAME }}

private_attribution_e2e_test:
runs-on: [self-hosted, e2e_test_runner]
needs: build_docker
steps:
- uses: actions/checkout@v2
- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull coordinator image
run: |
docker pull ${{ env.COORDINATOR_IMAGE_DEV }}

- name: Start container
run: |
./start_container.sh ${{ env.PA_CONTAINER_NAME }} ${{ env.COORDINATOR_IMAGE_DEV }}
working-directory: fbpcf/tests/github/

- name: Attribution - Create Instance
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} create_instance
working-directory: fbpcf/tests/github/

- name: Attribution - Data validation
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} data_validation
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - Id Match
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - PID export metrics
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} pid_metric_export
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - Prepare Compute Input
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} prepare_compute_input
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - PCF2 Attribution
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - PCF2 Aggregation
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - Aggregate Shards
run: |
./attribution_run_stages.sh ${{ env.PA_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PA_CONTAINER_NAME }} attribution
working-directory: fbpcf/tests/github/

- name: Attribution - Validate Result
run: |
./validate_result.sh attribution
working-directory: fbpcf/tests/github/

- name: Cleanup
run: |
docker stop ${{ env.PA_CONTAINER_NAME }}
docker rm ${{ env.PA_CONTAINER_NAME }}


private_lift_e2e_test:
runs-on: [self-hosted, e2e_test_runner]
needs: build_docker
steps:
- uses: actions/checkout@v2

- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull coordinator image
run: |
docker pull ${{ env.COORDINATOR_IMAGE_DEV }}

- name: Start container
run: |
./start_container.sh ${{ env.PL_CONTAINER_NAME }} ${{ env.COORDINATOR_IMAGE_DEV }}
working-directory: fbpcf/tests/github/

- name: Lift - Create Instance
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} create_instance
working-directory: fbpcf/tests/github/

- name: Lift - Data validation
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} data_validation
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Pid shard
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} pid_shard
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Pid prepare
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} pid_prepare
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Id Match
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - PID export metrics
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} pid_metric_export
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Prepare Compute Input
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} prepare_compute_input
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Compute Metrics
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Aggregate Shards
run: |
./lift_run_stages.sh ${{ env.PL_CONTAINER_NAME }} run_next
working-directory: fbpcf/tests/github/

- name: Check Status
timeout-minutes: 5
run: |
./check_status.sh ${{ env.PL_CONTAINER_NAME }} lift
working-directory: fbpcf/tests/github/

- name: Lift - Validate Results
run: |
./validate_result.sh lift
working-directory: fbpcf/tests/github/

- name: Cleanup
run: |
docker stop ${{ env.PL_CONTAINER_NAME }}
docker rm ${{ env.PL_CONTAINER_NAME }}