feat: Auto image build #157
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Push Preset Models | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'presets/falcon/**' | |
| - 'presets/llama-2/**' | |
| - 'presets/llama-2-chat/**' | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'presets/falcon/**' | |
| - 'presets/llama-2/**' | |
| - 'presets/llama-2-chat/**' | |
| workflow_dispatch: | |
| inputs: | |
| release: | |
| description: 'Release (yes/no)' | |
| required: true | |
| default: 'no' | |
| image_tag: | |
| description: 'Image Tag' | |
| required: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| setup: | |
| runs-on: self-hosted | |
| outputs: | |
| image_tag: ${{ steps.set_tag.outputs.image_tag }} | |
| FALCON_MODIFIED: ${{ steps.check_modified_paths.outputs.FALCON_MODIFIED }} | |
| LLAMA2_MODIFIED: ${{ steps.check_modified_paths.outputs.LLAMA2_MODIFIED }} | |
| LLAMA2_CHAT_MODIFIED: ${{ steps.check_modified_paths.outputs.LLAMA2_CHAT_MODIFIED }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| - name: Check Available Disk Space | |
| run: df -h | |
| - name: Get Modified files | |
| run: | | |
| files=$(git diff --name-only HEAD^ HEAD) | |
| echo "Modified files: $files" | |
| FILES_MODIFIED="" | |
| while IFS= read -r file; do | |
| trimmed_file=$(echo "$file" | tr -d '[:space:]') | |
| echo "Trimmed file: $trimmed_file" | |
| FILES_MODIFIED="${FILES_MODIFIED}${trimmed_file};" | |
| done <<< "$files" | |
| echo "FILES_MODIFIED=${FILES_MODIFIED}" >> $GITHUB_ENV | |
| - name: Check Modified Paths | |
| id: check_modified_paths | |
| run: | | |
| FALCON_MODIFIED=false | |
| LLAMA2_MODIFIED=false | |
| LLAMA2_CHAT_MODIFIED=false | |
| IFS=';' read -ra ADDR <<< "$FILES_MODIFIED" | |
| for file in "${ADDR[@]}"; do | |
| echo "Checking file: $file" | |
| if [[ "$file" == presets/falcon/* ]] && [[ "$FALCON_MODIFIED" == false ]]; then | |
| echo "File matches falcon path: $file" | |
| FALCON_MODIFIED=true | |
| elif [[ "$file" == presets/llama-2/* ]] && [[ "$LLAMA2_MODIFIED" == false ]]; then | |
| echo "File matches llama-2 path: $file" | |
| LLAMA2_MODIFIED=true | |
| elif [[ "$file" == presets/llama-2-chat/* ]] && [[ "$LLAMA2_CHAT_MODIFIED" == false ]]; then | |
| echo "File matches llama-2-chat path: $file" | |
| LLAMA2_CHAT_MODIFIED=true | |
| else | |
| echo "File does not match any paths: $file" | |
| fi | |
| done | |
| echo "FALCON_MODIFIED=$FALCON_MODIFIED" >> $GITHUB_OUTPUT | |
| echo "LLAMA2_MODIFIED=$LLAMA2_MODIFIED" >> $GITHUB_OUTPUT | |
| echo "LLAMA2_CHAT_MODIFIED=$LLAMA2_CHAT_MODIFIED" >> $GITHUB_OUTPUT | |
| - name: Models to Build | |
| run: | | |
| echo "FALCON_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.FALCON_MODIFIED }}" | |
| echo "LLAMA2_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.LLAMA2_MODIFIED }}" | |
| echo "LLAMA2_CHAT_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.LLAMA2_CHAT_MODIFIED }}" | |
| - name: Set Image Tag | |
| id: set_tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.image_tag }}" ]]; then | |
| echo "Using workflow dispatch to set image tag" | |
| echo "image_tag=${{ github.event.inputs.image_tag }}" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "Setting image tag to be latest" | |
| echo "image_tag=latest" >> $GITHUB_OUTPUT | |
| else | |
| echo "Setting image tag based on latest commit" | |
| echo "image_tag=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Print Image Tag | |
| run: | | |
| echo "image_tag for this job: ${{ steps.set_tag.outputs.image_tag }}" | |
| - name: Save image tag as artifact | |
| id: image_tag_path | |
| run: | | |
| echo "${{ steps.set_tag.outputs.image_tag }}" > image_tag.txt | |
| echo "IMAGE_TAG_PATH=$(pwd)/image_tag.txt" >> $GITHUB_OUTPUT | |
| - name: Upload image tag as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: image-tag-artifact | |
| path: ${{ steps.image_tag_path.outputs.IMAGE_TAG_PATH }} | |
| - name: Install Azure CLI latest | |
| run: | | |
| if ! which az > /dev/null; then | |
| echo "Azure CLI not found. Installing..." | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| else | |
| echo "Azure CLI already installed." | |
| fi | |
| matrix_prep: | |
| needs: setup | |
| runs-on: self-hosted | |
| outputs: | |
| matrix: ${{ steps.set_matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| - name: "Set matrix" | |
| id: set_matrix | |
| run: | | |
| matrix=$(jq --arg FALCON_MODIFIED "${{ needs.setup.outputs.FALCON_MODIFIED }}" --arg LLAMA2_MODIFIED "${{ needs.setup.outputs.LLAMA2_MODIFIED }}" --arg LLAMA2_CHAT_MODIFIED "${{ needs.setup.outputs.LLAMA2_CHAT_MODIFIED }}" 'map( | |
| . | select((.shouldBuildFalcon == $FALCON_MODIFIED) or (.shouldBuildLlama2 == $LLAMA2_MODIFIED) or (.shouldBuildLlama2Chat == $LLAMA2_CHAT_MODIFIED)) | |
| )' .github/matrix-configs.json) | |
| echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT | |
| build-models: | |
| needs: [setup, matrix_prep] | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| - name: 'Az CLI login' | |
| uses: azure/[email protected] | |
| with: | |
| client-id: ${{ secrets.AZURE_KDM_PRESET_SELF_RUNNER_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| allow-no-subscriptions: true | |
| - name: 'Set subscription' | |
| run: az account set --subscription ${{secrets.AZURE_SUBSCRIPTION_ID}} | |
| - name: 'Attach and Login to ACR' | |
| id: acr_login | |
| run: | | |
| if [[ "${{ needs.setup.outputs.image_tag }}" == "latest" ]]; then | |
| ACR_NAME="aimodelsregistry" | |
| else | |
| ACR_NAME="aimodelsregistrytest" | |
| fi | |
| CHECK_ACR_OUTPUT=$(az aks check-acr -g llm-test -n GitRunner --acr $ACR_NAME) | |
| if [[ ! $CHECK_ACR_OUTPUT =~ "Your cluster can pull images from $ACR_NAME.azurecr.io!" ]]; then | |
| az aks update -n GitRunner -g llm-test --attach-acr $ACR_NAME | |
| fi | |
| az acr login -n $ACR_NAME --expose-token | |
| echo "ACR_NAME=$ACR_NAME" >> $GITHUB_OUTPUT | |
| - name: Get Context | |
| run: az aks get-credentials -n GitRunner -g llm-test | |
| - name: Check if Docker Pod is Running (if not run it) | |
| run: | | |
| DEPLOYMENT=$(kubectl get deployment docker-deployment -o=jsonpath='{.metadata.name}' --ignore-not-found) | |
| if [ -z "$DEPLOYMENT" ]; then | |
| # Apply the deployment if it does not exist | |
| kubectl apply -f presets/k8s/docker.yaml | |
| fi | |
| kubectl wait --for=condition=ready pod -l app=docker --timeout=300s | |
| - name: Get Deployment Pod Name | |
| id: get_pod_name | |
| run: | | |
| POD_NAME=$(kubectl get pod -l app=docker -o=jsonpath='{.items[0].metadata.name}') | |
| echo "POD_NAME=$POD_NAME" >> $GITHUB_OUTPUT | |
| - name: Docker login | |
| run: | | |
| if [[ "${{ steps.acr_login.outputs.ACR_NAME }}" == "aimodelsregistry" ]]; then | |
| kubectl exec ${{ steps.get_pod_name.outputs.POD_NAME }} -- \ | |
| docker login ${{ steps.acr_login.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMR_USERNAME }} --password ${{ secrets.ACR_AMR_PASSWORD }} | |
| else | |
| kubectl exec ${{ steps.get_pod_name.outputs.POD_NAME }} -- \ | |
| docker login ${{ steps.acr_login.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMRT_USERNAME }} --password ${{ secrets.ACR_AMRT_PASSWORD }} | |
| fi | |
| - name: Build model (with retries) | |
| run: | | |
| retries=3 | |
| while [ $retries -gt 0 ]; do | |
| BUILD_ARGS="${{ matrix.model.build_args }}" | |
| echo "Docker BUILD_ARGS: $BUILD_ARGS" | |
| kubectl exec ${{ steps.get_pod_name.outputs.POD_NAME }} -- \ | |
| docker build \ | |
| $BUILD_ARGS \ | |
| -t ${{ steps.acr_login.outputs.ACR_NAME }}.azurecr.io/${{ matrix.model.name }}:${{ needs.setup.outputs.image_tag }} \ | |
| -f /home/${{ matrix.model.dockerfile }} \ | |
| . && break | |
| retries=$((retries-1)) | |
| sleep 15 | |
| done | |
| if [ $retries -eq 0 ]; then | |
| echo "Docker build failed after 3 retries." | |
| exit 1 | |
| fi | |
| - name: Push model to ACR (with retries) | |
| run: | | |
| retries=3 | |
| while [ $retries -gt 0 ]; do | |
| # Push the Docker image to ACR | |
| kubectl exec ${{ steps.get_pod_name.outputs.POD_NAME }} -- \ | |
| docker push ${{ steps.acr_login.outputs.ACR_NAME }}.azurecr.io/${{ matrix.model.name }}:${{ needs.setup.outputs.image_tag }} | |
| # Check if the push was successful | |
| if [ $? -eq 0 ]; then | |
| echo "Docker push successful!" | |
| break | |
| else | |
| echo "Docker push failed. Retrying..." | |
| retries=$((retries-1)) | |
| sleep 15 | |
| fi | |
| done | |
| if [ $retries -eq 0 ]; then | |
| echo "Docker push failed after 3 retries." | |
| exit 1 | |
| fi | |