ci: trigger #29
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy Spark Jobs | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'spark-jobs/**/*.py' | |
| jobs: | |
| build-and-deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # Fetch all history for git diff | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: buildx-${{ github.sha }} | |
| restore-keys: | | |
| buildx- | |
| - name: Get list of changed subfolders | |
| id: changed_subfolders | |
| run: | | |
| CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }}) | |
| echo "Changed files: $CHANGED_FILES" | |
| SUBFOLDERS=$(echo "$CHANGED_FILES" | grep '^spark-jobs/[^/]\+/.*\.py$' | awk -F/ '{print $2}' | sort | uniq | tr '\n' ' ') | |
| echo "SUBFOLDERS=$SUBFOLDERS" >> $GITHUB_ENV | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Build and deploy Docker images | |
| if: env.SUBFOLDERS != '' | |
| env: | |
| SUBFOLDERS: ${{ env.SUBFOLDERS }} | |
| run: | | |
| # Loop over each changed subfolder and build/push the Docker image | |
| for SUBFOLDER in $SUBFOLDERS; do | |
| echo "Processing subfolder: $SUBFOLDER" | |
| # Define image name and tag | |
| IMAGE_NAME="leothenardo/spark-$SUBFOLDER" | |
| IMAGE_TAG=${{ github.sha }} | |
| DOCKER_IMAGE="$IMAGE_NAME:$IMAGE_TAG" | |
| echo "Building Docker image: $DOCKER_IMAGE" | |
| # Build the Docker image with cache | |
| docker buildx build \ | |
| --cache-from=type=local,src=/tmp/.buildx-cache \ | |
| --cache-to=type=local,dest=/tmp/.buildx-cache,mode=max \ | |
| -t $DOCKER_IMAGE \ | |
| --push \ | |
| ./spark-jobs/$SUBFOLDER | |
| # Update the .yaml files in the subfolder | |
| echo "Updating YAML files in $SUBFOLDER" | |
| for YAML_FILE in ./spark-jobs/$SUBFOLDER/*.yaml; do | |
| echo "Updating YAML file: $YAML_FILE" | |
| sed -i "s|image: .*|image: \"$DOCKER_IMAGE\"|" "$YAML_FILE" | |
| done | |
| done | |
| - name: Commit and push changes | |
| if: env.SUBFOLDERS != '' | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: Update image references in YAML files [skip ci] | |
| file_pattern: spark-jobs/*/*.yaml | |
| branch: ${{ github.ref }} | |
| commit_options: '--no-verify --allow-empty' |