update

Signed-off-by: Sertac Ozercan <[email protected]>
sozercan · May 19, 2024 · e1e6bb4 · e1e6bb4
1 parent 6243ee4
commit e1e6bb4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 8 deletions.
diff --git a/pkg/aikit2llb/inference/convert.go b/pkg/aikit2llb/inference/convert.go
@@ -18,22 +18,23 @@ const (
 
 	localAIRepo    = "https://github.com/mudler/LocalAI"
 	localAIVersion = "v2.15.0"
+	// temporary commit until v2.16.0, used for python backends
 	localAICommit = "e2de8a88f70d18291eb34ceb035be79dc73d3be6"
-	cudaVersion    = "12-3"
+	cudaVersion   = "12-3"
 )
 
 func Aikit2LLB(c *config.InferenceConfig) (llb.State, *specs.Image) {
 	var merge llb.State
 	state := llb.Image(utils.DebianSlim)
-	// base := getBaseImage(c)
+	base := getBaseImage(c)
 
-	// state, merge = copyModels(c, base, state)
+	state, merge = copyModels(c, base, state)
 	state, merge = addLocalAI(c, state, merge)
 
-	// // install cuda if runtime is nvidia
-	// if c.Runtime == utils.RuntimeNVIDIA {
-	// 	state, merge = installCuda(c, state, merge)
-	// }
+	// install cuda if runtime is nvidia
+	if c.Runtime == utils.RuntimeNVIDIA {
+		state, merge = installCuda(c, state, merge)
+	}
 
 	// install opencv and friends if stable diffusion backend is being used
 	for b := range c.Backends {

diff --git a/pkg/aikit2llb/inference/exllama.go b/pkg/aikit2llb/inference/exllama.go
@@ -25,7 +25,7 @@ func installExllama(c *config.InferenceConfig, s llb.State, merge llb.State) llb
 	s = cloneLocalAI(s, backend)
 
 	// clone exllama to localai exllama backend path and install python dependencies
-	s = s.Run(utils.Bashf("git clone --depth 1 %[1]s --branch %[2]s /tmp/%[3]s && mv /tmp/%[3]s/* /tmp/localai/backend/python/%[3]s && rm -rf /tmp/%[3]s && cd /tmp/localai/backend/python/%[3]s && rm -rf .git && source $HOME/.cargo/env && python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto && uv venv && source .venv/bin/activate && ls -al && uv pip install --no-build-isolation --requirement requirements-install.txt && EXLLAMA_NOCOMPILE= uv pip install --no-build-isolation", exllamaRepo, exllamaTag, backend)).Root()
+	s = s.Run(utils.Bashf("git clone --depth 1 %[1]s --branch %[2]s /tmp/%[3]s && mv /tmp/%[3]s/* /tmp/localai/backend/python/%[3]s && rm -rf /tmp/%[3]s && cd /tmp/localai/backend/python/%[3]s && rm -rf .git && source $HOME/.cargo/env && python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto && uv venv && source .venv/bin/activate && ls -al && uv pip install --no-build-isolation --requirement requirements-install.txt && EXLLAMA_NOCOMPILE= uv pip install --no-build-isolation .", exllamaRepo, exllamaTag, backend)).Root()
 
 	diff := llb.Diff(savedState, s)
 	return llb.Merge([]llb.State{merge, diff})