From 718eda1b3ec79b9b4a5f87ac8350d6ed7f63598b Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Wed, 30 Apr 2025 11:25:22 -0700 Subject: [PATCH] Narrow set of paths we load GGML from (#10485) Users may have other incompatible GGML installs on their systems. This will prevent us from trying to load them from the path. --- llm/server.go | 7 ++++++- ml/backend/ggml/ggml/src/ggml.go | 14 ++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/llm/server.go b/llm/server.go index 82b514b2c..0f23fc4c5 100644 --- a/llm/server.go +++ b/llm/server.go @@ -329,11 +329,13 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...) } + ggmlPaths := []string{discover.LibOllamaPath} if len(compatible) > 0 { c := compatible[0] if libpath, ok := libs[c]; ok { slog.Debug("adding gpu library", "path", libpath) libraryPaths = append(libraryPaths, libpath) + ggmlPaths = append(ggmlPaths, libpath) } } @@ -369,6 +371,8 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a s.cmd.Stderr = s.status s.cmd.SysProcAttr = LlamaServerSysProcAttr + s.cmd.Env = append(s.cmd.Env, "OLLAMA_LIBRARY_PATH="+strings.Join(ggmlPaths, string(filepath.ListSeparator))) + envWorkarounds := [][2]string{} for _, gpu := range gpus { envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...) @@ -406,7 +410,8 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a if envconfig.Debug() { filteredEnv := []string{} for _, ev := range s.cmd.Env { - if strings.HasPrefix(ev, "CUDA_") || + if strings.HasPrefix(ev, "OLLAMA_") || + strings.HasPrefix(ev, "CUDA_") || strings.HasPrefix(ev, "ROCR_") || strings.HasPrefix(ev, "ROCM_") || strings.HasPrefix(ev, "HIP_") || diff --git a/ml/backend/ggml/ggml/src/ggml.go b/ml/backend/ggml/ggml/src/ggml.go index afc1e1edd..678f55ca6 100644 --- a/ml/backend/ggml/ggml/src/ggml.go +++ b/ml/backend/ggml/ggml/src/ggml.go @@ -57,26 +57,20 @@ var OnceLoad = sync.OnceFunc(func() { exe = "." } - // PATH, LD_LIBRARY_PATH, and DYLD_LIBRARY_PATH are often - // set by the parent process, however, use a default value - // if the environment variable is not set. - var name, value string + var value string switch runtime.GOOS { case "darwin": - // On macOS, DYLD_LIBRARY_PATH is often not set, so - // we use the directory of the executable as the default. - name = "DYLD_LIBRARY_PATH" value = filepath.Dir(exe) case "windows": - name = "PATH" value = filepath.Join(filepath.Dir(exe), "lib", "ollama") default: - name = "LD_LIBRARY_PATH" value = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama") } - paths, ok := os.LookupEnv(name) + // Avoid potentially loading incompatible GGML libraries + paths, ok := os.LookupEnv("OLLAMA_LIBRARY_PATH") if !ok { + slog.Debug("OLLAMA_LIBRARY_PATH not set, falling back to default", "search", value) paths = value }