feat: finetuning support

Signed-off-by: Sertac Ozercan <[email protected]>

.github/workflows/pre-release.yaml

@@ -59,7 +59,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
         id: build-and-push
         with:
           push: true

.github/workflows/release.yaml

@@ -67,7 +67,7 @@ jobs:
           tags: type=semver,pattern={{raw}}
 
       - name: Build and push
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
         id: build-and-push
         with:
           push: true

.github/workflows/test-docker-gpu.yaml

@@ -27,6 +27,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
         # use default docker driver builder with containerd image store for local aikit image
+        # these must be setup before running this test
       - run: docker buildx use default
 
       - name: build aikit

.github/workflows/test-finetune.yaml

@@ -9,37 +9,46 @@ jobs:
   test:
     runs-on: self-hosted
     timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        targets:
+          - unsloth
     steps:
       - name: cleanup workspace
         run: |
           rm -rf ./* || true
           rm -rf ./.??* || true
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+      - run: nvidia-smi
+
         # use default docker driver builder with containerd image store for local aikit image
         # must have insecure security entitlement for finetuning
         # these must be setup before running this test
       - run: docker buildx use default
 
       - name: build aikit
         run: |
-          docker build . -t aikit:test \
+          docker --debug build . -t aikit:test \
             --load --provenance=false --progress plain
 
       - name: build finetuned model
         run: |
-          docker build --allow security.insecure \
-            --file test/aikitfile-unsloth.yaml --output _output \
-            --target unsloth --progress plain .
+          docker --debug build --allow security.insecure \
+            --file test/aikitfile-${{ matrix.targets }}.yaml --output _output \
+            --target ${{ matrix.targets }} --progress plain .
 
       - name: check if finetuned model exists
-        run: test -f _output/model-q4_k_m.gguf
-
+        run: |
+          ls -al _output
+          test -f _output/model-q4_k_m.gguf
+
       - name: build custom model
-        working-directory: _output
         run: |
-          docker build . -t custommodel:test \
-            -f test/aikitfile-unsloth-custom.yaml \
+          docker --debug build _output --tag custommodel:test \
+            --file test/aikitfile-${{ matrix.targets }}-custom.yaml \
             --load --provenance=false --progress plain
 
       - name: list images
@@ -52,7 +61,7 @@ jobs:
         run: |
           result=$(curl --fail --retry 10 --retry-all-errors http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
             "model": "custom",
-            "messages": [{"role": "user", "content": "explain kubernetes in a sentence"}]
+            "messages": [{"role": "user", "content": "Compose a haiku about cats"}]
           }')
           echo $result
 
@@ -71,7 +80,7 @@ jobs:
       - run: docker system prune -a -f --volumes
         if: always()
 
-      - run: clean up output
+      - name: clean up output
         if: always()
         run: rm -rf _output
 

.gitignore

@@ -22,3 +22,5 @@ go.work
 
 bin
 coverage.txt
+
+_output

Dockerfile

@@ -1,7 +1,8 @@
-FROM golang:1.22-bookworm@sha256:925fe3fa28ba428cf67a7947ae838f8a1523117b40e3e6b5106c378e3f97fa29 as builder
+FROM golang:1.22-bookworm@sha256:6699d2852712f090399ccd4e8dfd079b4d55376f3ab3aff5b2dc8b7b1c11e27e as builder
+ARG LDFLAGS
 COPY . /go/src/github.com/sozercan/aikit
 WORKDIR /go/src/github.com/sozercan/aikit
-RUN CGO_ENABLED=0 go build -o /aikit --ldflags '-extldflags "-static"' ./cmd/frontend
+RUN CGO_ENABLED=0 go build -o /aikit -ldflags "${LDFLAGS} -extldflags '-static'" ./cmd/frontend
 
 FROM scratch
 COPY --from=builder /aikit /bin/aikit

Makefile

@@ -6,13 +6,17 @@ OUTPUT_TYPE ?= type=docker
 TEST_IMAGE_NAME ?= testmodel
 TEST_FILE ?= test/aikitfile-llama.yaml
 
+GIT_COMMIT := $(shell git rev-list --abbrev-commit --tags --max-count=1)
+GIT_TAG := $(shell git describe --abbrev=0 --tags ${GIT_COMMIT} 2>/dev/null || true)
+LDFLAGS := "-X github.com/sozercan/aikit/pkg/version.Version=$(GIT_TAG:%=%)"
+
 .PHONY: lint
 lint:
 	golangci-lint run -v ./... --timeout 5m
 
 .PHONY: build-aikit
 build-aikit:
-	docker buildx build . -t ${REGISTRY}/aikit:${TAG} --output=${OUTPUT_TYPE}
+	docker buildx build . -t ${REGISTRY}/aikit:${TAG} --output=${OUTPUT_TYPE} --build-arg LDFLAGS=${LDFLAGS}
 
 .PHONY: build-test-model
 build-test-model:

demo/demo.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+. third_party/demo-magic/demo-magic.sh
+
+clear
+DEMO_PROMPT="${GREEN}➜  ${COLOR_RESET}"
+
+echo "✨ In this demo, we are going to start by fine tuning a model and then deploy the model as a minimal container!"
+
+echo ""
+
+echo "👷‍ First, we are going to create a new builder"
+
+echo ""
+
+pei "docker buildx create --name aikit-builder --use --buildkitd-flags '--allow-insecure-entitlement security.insecure'"
+
+echo ""
+
+echo "🗃️ Create a configuration for the fine tuning. We are going to be using a Mistral model and fine tune using OpenHermes dataset."
+
+cat > aikit-finetune.yaml << EOF
+#syntax=ghcr.io/sozercan/aikit:latest
+apiVersion: v1alpha1
+baseModel: unsloth/mistral-7b-instruct-v0.2-bnb-4bit
+datasets:
+  - source: "teknium/openhermes"
+    type: "alpaca"
+config:
+  unsloth:
+EOF
+
+echo ""
+
+pei "bat aikit-finetune.yaml"
+
+echo ""
+
+echo "🎵 Starting the fine tuning process using the above configuration file, and output fine tuned model will be saved in _output folder."
+
+echo ""
+
+pei "docker buildx build --allow security.insecure --file 'aikit-finetune.yaml' --output '_output' --target unsloth --progress plain ."
+
+echo ""
+
+echo "✅ We have finished fine tuning the model. Let's look at the output..."
+
+echo ""
+
+pei "ls -al _output"
+
+echo ""
+
+echo "📦 Now that we have a fine tuned model. We can deploy it as a minimal container."
+
+echo ""
+
+echo "📃 We'll start by creating a basic inference configuration file for the deployment."
+
+cat > aikit-inference.yaml << EOF
+#syntax=ghcr.io/sozercan/aikit:latest
+debug: true
+apiVersion: v1alpha1
+runtime: cuda
+models:
+  - name: mistral-finetuned
+    source: aikit-model-q4_k_m.gguf
+    promptTemplates:
+      - name: instruct
+        template: |
+          Below is an instruction that describes a task. Write a response that appropriately completes the request. Keep your responses concise.
+
+          ### Instruction:
+          {{.Input}}
+
+          ### Response:
+config: |
+  - name: mistral-finetuned
+    parameters:
+      model: aikit-model-q4_k_m.gguf
+    context_size: 4096
+    gpu_layers: 35
+    f16: true
+    batch: 512
+    mmap: true
+    template:
+      chat: instruct
+EOF
+
+pei "bat aikit-inference.yaml"
+
+echo ""
+
+echo "🏗️ We can now build a minimal container for the model using the configuration file."
+
+echo ""
+
+pei "docker buildx build -t mistral-finetuned -f aikit-inference.yaml --load --progress plain _output"
+
+echo ""
+
+echo "🏃 We have finished building the minimal container. Let's start the container and test it."
+
+echo ""
+
+pei "docker run --name mistral-finetuned -d --rm -p 8080:8080 --gpus all mistral-finetuned"
+
+echo ""
+
+echo "🧪 We can now test the container using a sample query. Since this is OpenAI API compatible, you can use it as a drop-in replacement for any application that uses OpenAI API."
+
+echo ""
+
+pei "curl http://localhost:8080/v1/chat/completions -H \"Content-Type: application/json\" -d '{\"model\": \"mistral-finetuned\", \"messages\": [{\"role\": \"user\", \"content\": \"Generate a list of 10 words that start with ab\"}]}'"
+
+echo ""
+
+echo "🙌 We have successfully deployed the fine tuned model as a minimal container and successfully verified it! We can now stop the container if we wish."
+
+echo ""
+
+pei "docker stop mistral-finetuned"
+
+echo ""
+
+echo "❤️ In this demo, we have shown how to fine tune a model and deploy it as a minimal container using AIKit. Thank you for watching!"
+
+echo ""
+
+# pei "docker buildx rm aikit-builder"