jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tensor count

Browse Source
This commit is contained in:
Josh Yan 2024-07-09 11:02:58 -07:00
parent 1344843515
commit bec9100f32
1 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server/images.go
View File

@ -413,8 +413,7 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
return fmt.Errorf("invalid model reference: %s", c.Args) return fmt.Errorf("invalid model reference: %s", c.Args)
} }
layerCount := len(baseLayers) for _, baseLayer := range baseLayers {
for i, baseLayer := range baseLayers {
if quantization != "" && if quantization != "" &&
baseLayer.MediaType == "application/vnd.ollama.image.model" && baseLayer.MediaType == "application/vnd.ollama.image.model" &&
baseLayer.GGML != nil && baseLayer.GGML != nil &&
@ -424,6 +423,12 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
return err return err
} }
tensorCount := len(baseLayer.GGML.Tensors())
fn(api.ProgressResponse{
Status: fmt.Sprintf("quantizing model %d tensors", tensorCount),
Quantize: quantization,
})
ft := baseLayer.GGML.KV().FileType() ft := baseLayer.GGML.KV().FileType()
if !slices.Contains([]string{"F16", "F32"}, ft.String()) { if !slices.Contains([]string{"F16", "F32"}, ft.String()) {
return errors.New("quantization is only supported for F16 and F32 models") return errors.New("quantization is only supported for F16 and F32 models")
@ -463,6 +468,11 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
baseLayer.Layer = layer baseLayer.Layer = layer
baseLayer.GGML = ggml baseLayer.GGML = ggml
} }
fn(api.ProgressResponse{
Status: "quantizing model done",
Quantize: quantization,
})
} }
if baseLayer.GGML != nil { if baseLayer.GGML != nil {
@ -473,18 +483,14 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
config.ModelFamilies = append(config.ModelFamilies, baseLayer.GGML.KV().Architecture()) config.ModelFamilies = append(config.ModelFamilies, baseLayer.GGML.KV().Architecture())
} }
fn(api.ProgressResponse{ /* fn(api.ProgressResponse{
Status: fmt.Sprintf("quantizing model %d%%", i*100/layerCount), Status: fmt.Sprintf("quantizing model %d%%", i*100/layerCount),
Quantize: quantization, Quantize: quantization,
}) }) */
layers = append(layers, baseLayer.Layer) layers = append(layers, baseLayer.Layer)
} }
fn(api.ProgressResponse{
Status: "quantizing model done",
Quantize: quantization,
})
case "license", "template", "system": case "license", "template", "system":
if c.Name != "license" { if c.Name != "license" {
// replace // replace