From ad035ad595295d0026a5a94f8180962bbf0fa935 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Mon, 12 May 2025 12:04:20 -0700 Subject: [PATCH] convert: quantize from safetensors needs kv (#10675) When creating a quantized model from safetensors we need the array KV values to be loaded.Changing this value to -1 loads the KV values on the returned layer to be used and saved during quantization. --- server/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/create.go b/server/create.go index 7ffa60a22..41c8731cc 100644 --- a/server/create.go +++ b/server/create.go @@ -295,7 +295,7 @@ func convertFromSafetensors(files map[string]string, baseLayers []*layerGGML, is } defer bin.Close() - f, _, err := ggml.Decode(bin, 1024) + f, _, err := ggml.Decode(bin, -1) if err != nil { return nil, err }