ci: trigger ci #13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy Spark Jobs | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'spark-jobs/**.py' | |
| jobs: | |
| build-and-deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Get list of changed subfolders | |
| id: changed_subfolders | |
| run: | | |
| CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }}) | |
| echo "Changed files: $CHANGED_FILES" | |
| SUBFOLDERS=$(echo "$CHANGED_FILES" | grep '^spark-jobs/[^/]\+/.\+\.py$' | awk -F/ '{print $2}' | sort | uniq) | |
| echo "SUBFOLDERS=$SUBFOLDERS" >> $GITHUB_ENV | |
| - name: Build and deploy Docker images | |
| if: env.SUBFOLDERS != '' | |
| env: | |
| SUBFOLDERS: ${{ env.SUBFOLDERS }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| run: | | |
| echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin | |
| for SUBFOLDER in $SUBFOLDERS; do | |
| echo "Processing subfolder: $SUBFOLDER" | |
| # Define image name and tag | |
| IMAGE_NAME="leothenardo/spark-$SUBFOLDER" | |
| IMAGE_TAG=${{ github.sha }} | |
| DOCKER_IMAGE="$IMAGE_NAME:$IMAGE_TAG" | |
| echo "Building Docker image: $DOCKER_IMAGE" | |
| # Build the Docker image | |
| docker build -t $DOCKER_IMAGE ./spark-jobs/$SUBFOLDER | |
| # Push the image | |
| docker push $DOCKER_IMAGE | |
| # Update the .yaml files in the subfolder | |
| echo "Updating YAML files in $SUBFOLDER" | |
| for YAML_FILE in ./spark-jobs/$SUBFOLDER/*.yaml; do | |
| echo "Updating YAML file: $YAML_FILE" | |
| sed -i "s|image: .*|image: \"$DOCKER_IMAGE\"|" "$YAML_FILE" | |
| done | |
| done | |
| - name: Commit and push changes | |
| if: env.SUBFOLDERS != '' | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: Update image references in YAML files [skip ci] | |
| file_pattern: spark-jobs/*/*.yaml | |
| branch: ${{ github.ref }} | |
| commit_options: '--no-verify --allow-empty' |