diff --git a/pkg/aikit2llb/inference/convert.go b/pkg/aikit2llb/inference/convert.go index c7f81d2c..15bf8bd5 100644 --- a/pkg/aikit2llb/inference/convert.go +++ b/pkg/aikit2llb/inference/convert.go @@ -18,22 +18,23 @@ const ( localAIRepo = "https://github.com/mudler/LocalAI" localAIVersion = "v2.15.0" + // temporary commit until v2.16.0, used for python backends localAICommit = "e2de8a88f70d18291eb34ceb035be79dc73d3be6" - cudaVersion = "12-3" + cudaVersion = "12-3" ) func Aikit2LLB(c *config.InferenceConfig) (llb.State, *specs.Image) { var merge llb.State state := llb.Image(utils.DebianSlim) - // base := getBaseImage(c) + base := getBaseImage(c) - // state, merge = copyModels(c, base, state) + state, merge = copyModels(c, base, state) state, merge = addLocalAI(c, state, merge) - // // install cuda if runtime is nvidia - // if c.Runtime == utils.RuntimeNVIDIA { - // state, merge = installCuda(c, state, merge) - // } + // install cuda if runtime is nvidia + if c.Runtime == utils.RuntimeNVIDIA { + state, merge = installCuda(c, state, merge) + } // install opencv and friends if stable diffusion backend is being used for b := range c.Backends { diff --git a/pkg/aikit2llb/inference/exllama.go b/pkg/aikit2llb/inference/exllama.go index feb70bb3..06e1569f 100644 --- a/pkg/aikit2llb/inference/exllama.go +++ b/pkg/aikit2llb/inference/exllama.go @@ -25,7 +25,7 @@ func installExllama(c *config.InferenceConfig, s llb.State, merge llb.State) llb s = cloneLocalAI(s, backend) // clone exllama to localai exllama backend path and install python dependencies - s = s.Run(utils.Bashf("git clone --depth 1 %[1]s --branch %[2]s /tmp/%[3]s && mv /tmp/%[3]s/* /tmp/localai/backend/python/%[3]s && rm -rf /tmp/%[3]s && cd /tmp/localai/backend/python/%[3]s && rm -rf .git && source $HOME/.cargo/env && python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto && uv venv && source .venv/bin/activate && ls -al && uv pip install --no-build-isolation --requirement requirements-install.txt && EXLLAMA_NOCOMPILE= uv pip install --no-build-isolation", exllamaRepo, exllamaTag, backend)).Root() + s = s.Run(utils.Bashf("git clone --depth 1 %[1]s --branch %[2]s /tmp/%[3]s && mv /tmp/%[3]s/* /tmp/localai/backend/python/%[3]s && rm -rf /tmp/%[3]s && cd /tmp/localai/backend/python/%[3]s && rm -rf .git && source $HOME/.cargo/env && python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto && uv venv && source .venv/bin/activate && ls -al && uv pip install --no-build-isolation --requirement requirements-install.txt && EXLLAMA_NOCOMPILE= uv pip install --no-build-isolation .", exllamaRepo, exllamaTag, backend)).Root() diff := llb.Diff(savedState, s) return llb.Merge([]llb.State{merge, diff})