jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

check total (system + video) memory

Browse Source
This commit is contained in:
Michael Yang 2023-10-12 10:36:23 -07:00
parent bd6e38fb1a
commit 4a8931f634
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
llm/llm.go
View File

@ -60,7 +60,7 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
var requiredMemory int64 var requiredMemory int64
var f16Multiplier int64 = 2 var f16Multiplier int64 = 2
totalResidentMemory := int64(memory.TotalMemory())
switch ggml.ModelType() { switch ggml.ModelType() {
case "3B", "7B": case "3B", "7B":
requiredMemory = 8 * format.GigaByte requiredMemory = 8 * format.GigaByte
@ -75,10 +75,19 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
f16Multiplier = 4 f16Multiplier = 4
} }
if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > totalResidentMemory { systemMemory := int64(memory.TotalMemory())
return nil, fmt.Errorf("F16 model requires at least %s of memory", format.HumanBytes(requiredMemory))
} else if requiredMemory > totalResidentMemory { videoMemory, err := CheckVRAM()
return nil, fmt.Errorf("model requires at least %s of memory", format.HumanBytes(requiredMemory)) if err != nil{
videoMemory = 0
}
totalMemory := systemMemory + videoMemory
if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > totalMemory {
return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} else if requiredMemory > totalMemory {
return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} }
switch ggml.Name() { switch ggml.Name() {