test

Signed-off-by: Sertac Ozercan <[email protected]>
sozercan · Jan 25, 2024 · 336d5d0 · 336d5d0
1 parent 28ac0cc
commit 336d5d0
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 9 deletions.
diff --git a/.github/workflows/test-docker-gpu.yaml b/.github/workflows/test-docker-gpu.yaml
@@ -14,9 +14,9 @@ jobs:
       max-parallel: 1
       matrix:
         backend:
-          - llama-cuda
-          # - exllama # https://github.com/sozercan/aikit/issues/94
-          - exllama2-gptq
+          # - llama-cuda
+          # # - exllama # https://github.com/sozercan/aikit/issues/94
+          # - exllama2-gptq
           - exllama2-exl2
           - mamba
     steps:
@@ -44,15 +44,24 @@ jobs:
         run: docker run --name testmodel -d --rm -p 8080:8080 --gpus all testmodel:test
 
       - name: run test
+        if: matrix.backend != 'mamba'
         run: |
-          if [ "${{ matrix.backend }}" == "mamba" ]; then
-            MODEL_NAME="mamba-chat"
-          else
-            MODEL_NAME="llama-2-7b-chat"
+          result=$(curl --fail --retry 10 --retry-all-errors http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+            "model": "llama-2-7b-chat",
+            "messages": [{"role": "user", "content": "explain kubernetes in a sentence"}]
+          }')
+          echo $result
+
+          choices=$(echo "$result" | jq '.choices')
+          if [ -z "$choices" ]; then
+            exit 1
           fi
 
+      - name: run test
+        if: matrix.backend == 'mamba'
+        run: |
           result=$(curl --fail --retry 10 --retry-all-errors http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
-            "model": ${MODEL_NAME},
+            "model": "mamba-chat",
             "messages": [{"role": "user", "content": "explain kubernetes in a sentence"}]
           }')
           echo $result

diff --git a/pkg/aikit2llb/convert.go b/pkg/aikit2llb/convert.go
@@ -17,7 +17,7 @@ const (
 	debianSlim     = "docker.io/library/debian:12-slim"
 	distrolessBase = "gcr.io/distroless/cc-debian12:latest"
 
-	localAIVersion = "v2.6.0"
+	localAIVersion = "v2.6.1"
 	localAIRepo    = "https://github.com/mudler/LocalAI"
 	cudaVersion    = "12-3"
 )