sample: do all sorting in topK

README.md

@@ -54,10 +54,6 @@ Here are some example models that can be downloaded:
 
 | Model              | Parameters | Size  | Download                         |
 | ------------------ | ---------- | ----- | -------------------------------- |
-| Gemma 3            | 1B         | 815MB | `ollama run gemma3:1b`           |
-| Gemma 3            | 4B         | 3.3GB | `ollama run gemma3`              |
-| Gemma 3            | 12B        | 8.1GB | `ollama run gemma3:12b`          |
-| Gemma 3            | 27B        | 17GB  | `ollama run gemma3:27b`          |
 | QwQ                | 32B        | 20GB  | `ollama run qwq`                 |
 | DeepSeek-R1        | 7B         | 4.7GB | `ollama run deepseek-r1`         |
 | DeepSeek-R1        | 671B       | 404GB | `ollama run deepseek-r1:671b`    |
@@ -70,6 +66,9 @@ Here are some example models that can be downloaded:
 | Llama 3.1          | 405B       | 231GB | `ollama run llama3.1:405b`       |
 | Phi 4              | 14B        | 9.1GB | `ollama run phi4`                |
 | Phi 4 Mini         | 3.8B       | 2.5GB | `ollama run phi4-mini`           |
+| Gemma 2            | 2B         | 1.6GB | `ollama run gemma2:2b`           |
+| Gemma 2            | 9B         | 5.5GB | `ollama run gemma2`              |
+| Gemma 2            | 27B        | 16GB  | `ollama run gemma2:27b`          |
 | Mistral            | 7B         | 4.1GB | `ollama run mistral`             |
 | Moondream 2        | 1.4B       | 829MB | `ollama run moondream`           |
 | Neural Chat        | 7B         | 4.1GB | `ollama run neural-chat`         |

api/types.go

@@ -349,7 +349,6 @@ type ShowResponse struct {
 	Messages      []Message      `json:"messages,omitempty"`
 	ModelInfo     map[string]any `json:"model_info,omitempty"`
 	ProjectorInfo map[string]any `json:"projector_info,omitempty"`
-	Tensors       []Tensor       `json:"tensors,omitempty"`
 	ModifiedAt    time.Time      `json:"modified_at,omitempty"`
 }
 
@@ -468,13 +467,6 @@ type ModelDetails struct {
 	QuantizationLevel string   `json:"quantization_level"`
 }
 
-// Tensor describes the metadata for a given tensor.
-type Tensor struct {
-	Name  string   `json:"name"`
-	Type  string   `json:"type"`
-	Shape []uint64 `json:"shape"`
-}
-
 func (m *Metrics) Summary() {
 	if m.TotalDuration > 0 {
 		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)

cmd/cmd.go

@@ -18,7 +18,6 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -569,9 +568,8 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 	parameters, errParams := cmd.Flags().GetBool("parameters")
 	system, errSystem := cmd.Flags().GetBool("system")
 	template, errTemplate := cmd.Flags().GetBool("template")
-	verbose, errVerbose := cmd.Flags().GetBool("verbose")
 
-	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate, errVerbose} {
+	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
 		if boolErr != nil {
 			return errors.New("error retrieving flags")
 		}
@@ -609,7 +607,7 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
 	}
 
-	req := api.ShowRequest{Name: args[0], Verbose: verbose}
+	req := api.ShowRequest{Name: args[0]}
 	resp, err := client.Show(cmd.Context(), &req)
 	if err != nil {
 		return err
@@ -632,10 +630,10 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	return showInfo(resp, verbose, os.Stdout)
+	return showInfo(resp, os.Stdout)
 }
 
-func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
+func showInfo(resp *api.ShowResponse, w io.Writer) error {
 	tableRender := func(header string, rows func() [][]string) {
 		fmt.Fprintln(w, " ", header)
 		table := tablewriter.NewWriter(w)
@@ -692,45 +690,6 @@ func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
 		})
 	}
 
-	if resp.ModelInfo != nil && verbose {
-		tableRender("Metadata", func() (rows [][]string) {
-			keys := make([]string, 0, len(resp.ModelInfo))
-			for k := range resp.ModelInfo {
-				keys = append(keys, k)
-			}
-			sort.Strings(keys)
-
-			for _, k := range keys {
-				var v string
-				switch vData := resp.ModelInfo[k].(type) {
-				case string:
-					v = vData
-				case float64:
-					v = fmt.Sprintf("%g", vData)
-				case []any:
-					n := 3
-					if len(vData) < n {
-						n = len(vData)
-					}
-					v = fmt.Sprintf("%v", vData[:n])
-				default:
-					v = fmt.Sprintf("%T", vData)
-				}
-				rows = append(rows, []string{"", k, v})
-			}
-			return
-		})
-	}
-
-	if len(resp.Tensors) > 0 && verbose {
-		tableRender("Tensors", func() (rows [][]string) {
-			for _, t := range resp.Tensors {
-				rows = append(rows, []string{"", t.Name, t.Type, fmt.Sprint(t.Shape)})
-			}
-			return
-		})
-	}
-
 	head := func(s string, n int) (rows [][]string) {
 		scanner := bufio.NewScanner(strings.NewReader(s))
 		for scanner.Scan() && (len(rows) < n || n < 0) {
@@ -1237,7 +1196,6 @@ func NewCLI() *cobra.Command {
 	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
 	showCmd.Flags().Bool("template", false, "Show template of a model")
 	showCmd.Flags().Bool("system", false, "Show system message of a model")
-	showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")
 
 	runCmd := &cobra.Command{
 		Use:     "run MODEL [PROMPT]",

cmd/cmd_test.go

@@ -27,7 +27,7 @@ func TestShowInfo(t *testing.T) {
 				ParameterSize:     "7B",
 				QuantizationLevel: "FP16",
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -57,7 +57,7 @@ func TestShowInfo(t *testing.T) {
 				ParameterSize:     "7B",
 				QuantizationLevel: "FP16",
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -68,56 +68,6 @@ func TestShowInfo(t *testing.T) {
     embedding length    0       
     quantization        FP16    
 
-`
-		if diff := cmp.Diff(expect, b.String()); diff != "" {
-			t.Errorf("unexpected output (-want +got):\n%s", diff)
-		}
-	})
-
-	t.Run("verbose model", func(t *testing.T) {
-		var b bytes.Buffer
-		if err := showInfo(&api.ShowResponse{
-			Details: api.ModelDetails{
-				Family:            "test",
-				ParameterSize:     "8B",
-				QuantizationLevel: "FP16",
-			},
-			Parameters: `
-			stop up`,
-			ModelInfo: map[string]any{
-				"general.architecture":    "test",
-				"general.parameter_count": float64(8_000_000_000),
-				"test.context_length":     float64(1000),
-				"test.embedding_length":   float64(11434),
-			},
-			Tensors: []api.Tensor{
-				{Name: "blk.0.attn_k.weight", Type: "BF16", Shape: []uint64{42, 3117}},
-				{Name: "blk.0.attn_q.weight", Type: "FP16", Shape: []uint64{3117, 42}},
-			},
-		}, true, &b); err != nil {
-			t.Fatal(err)
-		}
-
-		expect := `  Model
-    architecture        test     
-    parameters          8B       
-    context length      1000     
-    embedding length    11434    
-    quantization        FP16     
-
-  Parameters
-    stop    up    
-
-  Metadata
-    general.architecture       test     
-    general.parameter_count    8e+09    
-    test.context_length        1000     
-    test.embedding_length      11434    
-
-  Tensors
-    blk.0.attn_k.weight    BF16    [42 3117]    
-    blk.0.attn_q.weight    FP16    [3117 42]    
-
 `
 		if diff := cmp.Diff(expect, b.String()); diff != "" {
 			t.Errorf("unexpected output (-want +got):\n%s", diff)
@@ -139,7 +89,7 @@ func TestShowInfo(t *testing.T) {
 			stop you
 			stop up
 			temperature 99`,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -176,7 +126,7 @@ func TestShowInfo(t *testing.T) {
 				"clip.vision.embedding_length": float64(0),
 				"clip.vision.projection_dim":   float64(0),
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -209,7 +159,7 @@ func TestShowInfo(t *testing.T) {
 Ahoy, matey!
 Weigh anchor!
 			`,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -238,7 +188,7 @@ Weigh anchor!
 				QuantizationLevel: "FP16",
 			},
 			License: license,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 

cmd/interactive.go

@@ -195,10 +195,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 			opts.Messages = []api.Message{}
 			fmt.Printf("Loading model '%s'\n", opts.Model)
 			if err := loadOrUnloadModel(cmd, &opts); err != nil {
-				if strings.Contains(err.Error(), "not found") {
-					fmt.Printf("error: %v\n", err)
-					continue
-				}
 				return err
 			}
 			continue
@@ -347,7 +343,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 
 				switch args[1] {
 				case "info":
-					_ = showInfo(resp, false, os.Stderr)
+					_ = showInfo(resp, os.Stderr)
 				case "license":
 					if resp.License == "" {
 						fmt.Println("No license was specified for this model.")

convert/convert_gemma3.go

@@ -87,7 +87,7 @@ func (p *gemma3Model) KV(t *Tokenizer) ggml.KV {
 		kv["gemma3.embedding_length"] = p.HiddenSize
 		kv["gemma3.feed_forward_length"] = p.IntermediateSize
 	default:
-		kv["gemma3.context_length"] = cmp.Or(p.MaxPositionEmbeddings, 131072)
+		kv["gemma3.context_length"] = cmp.Or(p.MaxPositionEmbeddings, 8192)
 		kv["gemma3.embedding_length"] = p.TextModel.HiddenSize
 		kv["gemma3.feed_forward_length"] = p.TextModel.IntermediateSize
 		kv["gemma3.attention.sliding_window"] = p.TextModel.SlidingWindow

fs/ggml/ggml.go

@@ -327,10 +327,6 @@ func (t Tensor) Size() uint64 {
 	return t.parameters() * t.typeSize() / t.blockSize()
 }
 
-func (t Tensor) Type() string {
-	return fileType(t.Kind).String()
-}
-
 type container interface {
 	Name() string
 	Decode(io.ReadSeeker) (model, error)

logging/log.go

@@ -1,40 +0,0 @@
-package logging
-
-import (
-	"context"
-	"log/slog"
-	"os"
-)
-
-const LevelTrace slog.Level = slog.LevelDebug - 4
-
-type Logger struct {
-	logger *slog.Logger
-}
-
-func NewLogger() *Logger {
-	handler := slog.NewTextHandler(os.Stdout, nil)
-	return &Logger{
-		logger: slog.New(handler),
-	}
-}
-
-func (l *Logger) Trace(msg string, args ...any) {
-	l.logger.Log(context.Background(), LevelTrace, msg, args...)
-}
-
-func (l *Logger) Debug(msg string, args ...any) {
-	l.logger.Debug(msg, args...)
-}
-
-func (l *Logger) Info(msg string, args ...any) {
-	l.logger.Info(msg, args...)
-}
-
-func (l *Logger) Warn(msg string, args ...any) {
-	l.logger.Warn(msg, args...)
-}
-
-func (l *Logger) Error(msg string, args ...any) {
-	l.logger.Error(msg, args...)
-}

ml/backend/ggml/ggml/src/ollama-debug.c

@@ -1,5 +1,4 @@
 #include <string.h>
-#include <inttypes.h>
 
 #include "ollama-debug.h"
 
@@ -25,7 +24,7 @@ static void print_tensor(const void *tensor, void (*cb)(const void *, int),
     fprintf(stderr, "[");
     for (int i = 0; i < dims[0]; i++) {
         if (i >= nitems && i < dims[0] - nitems) {
-            fprintf(stderr, "... (%" PRIi64 " more), ", dims[0] - 2 * nitems);
+            fprintf(stderr, "... (%lld more), ", dims[0] - 2 * nitems);
             int skip = dims[0] - 2 * nitems;
             if (ndims > 1) {
                 stride += mul(dims + 1, ndims - 1) * skip;
@@ -68,7 +67,7 @@ static void print_tensor_i32(const void *tensor, int i) {
 }
 
 static void ollama_debug_tensor(const struct ggml_tensor *tensor, bool verbose, const char *prefix, int indent) {
-    fprintf(stderr, "%s%s %s (%s): [%" PRIi64 " %" PRIi64 " %" PRIi64 " %" PRIi64 "]\n", prefix, tensor->name,
+    fprintf(stderr, "%s%s %s (%s): [%lld %lld %lld %lld]\n", prefix, tensor->name,
             ggml_op_name(tensor->op), ggml_type_name(tensor->type), tensor->ne[0],
             tensor->ne[1], tensor->ne[2], tensor->ne[3]);
 

model/model.go

@@ -22,8 +22,6 @@ import (
 	"github.com/ollama/ollama/model/input"
 )
 
-var ErrNoVisionModel = errors.New("this model is missing data required for image input")
-
 // Model implements a specific model architecture, defining the forward pass and any model-specific configuration
 type Model interface {
 	Forward(ml.Context, input.Options) (ml.Tensor, error)

model/models/gemma3/model.go

@@ -84,10 +84,6 @@ func New(c ml.Config) (model.Model, error) {
 }
 
 func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
-	if len(m.VisionModel.Layers) == 0 {
-		return nil, model.ErrNoVisionModel
-	}
-
 	image, _, err := image.Decode(bytes.NewReader(multimodalData))
 	if err != nil {
 		return nil, err

model/models/gemma3/model_text.go

@@ -15,6 +15,7 @@ type TextOptions struct {
 	attnKeyLen, attnValLen           int
 	eps, ropeScale                   float32
 	ropeLocalBase, ropeGlobalBase    float32
+	finalLogitSoftcap                float32
 	largeModelScaling                bool
 }
 
@@ -65,6 +66,7 @@ func newTextModel(c ml.Config) *TextModel {
 			ropeLocalBase:     c.Float("rope.local.freq_base", 10000.0),
 			ropeGlobalBase:    c.Float("rope.global.freq_base", 1000000.0),
 			ropeScale:         c.Float("rope.freq_scale", 1.0),
+			finalLogitSoftcap: c.Float("final_logit_softcapping", 30.0),
 		},
 	}
 
@@ -243,5 +245,10 @@ func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor
 	}
 
 	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	return m.Output.Forward(ctx, hiddenState)
+	hiddenState = m.Output.Forward(ctx, hiddenState)
+
+	// final logit softcap
+	hiddenState = hiddenState.Scale(ctx, 1.0/float64(m.TextOptions.finalLogitSoftcap))
+	hiddenState = hiddenState.Tanh(ctx)
+	return hiddenState.Scale(ctx, float64(m.TextOptions.finalLogitSoftcap))
 }

model/models/mllama/model.go

@@ -63,10 +63,6 @@ func New(c ml.Config) (model.Model, error) {
 }
 
 func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
-	if len(m.VisionModel.Transformer.Layers) == 0 || len(m.GlobalTransformer.Layers) == 0 {
-		return nil, model.ErrNoVisionModel
-	}
-
 	image, _, err := image.Decode(bytes.NewReader(multimodalData))
 	if err != nil {
 		return nil, err

model/process_text_spm.go

@@ -2,18 +2,15 @@ package model
 
 import (
 	"iter"
+	"log/slog"
 	"strings"
 
 	"github.com/dlclark/regexp2"
 	queue "github.com/emirpasic/gods/v2/queues/priorityqueue"
-
-	"github.com/ollama/ollama/logging"
 )
 
 const spmWhitespaceSep = "▁"
 
-var log = logging.NewLogger()
-
 func replaceWhitespaceBySeperator(s string) string {
 	return strings.ReplaceAll(s, " ", spmWhitespaceSep)
 }
@@ -27,7 +24,7 @@ type SentencePieceModel struct {
 var _ TextProcessor = (*SentencePieceModel)(nil)
 
 func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel {
-	log.Debug("Tokens", "num tokens", len(vocab.Values), "vals", vocab.Values[:5], "scores", vocab.Scores[:5], "types", vocab.Types[:5])
+	slog.Debug("Tokens", "num tokens", len(vocab.Values), "vals", vocab.Values[:5], "scores", vocab.Scores[:5], "types", vocab.Types[:5])
 
 	counter := map[int]int{}
 	var maxTokenLen int
@@ -41,7 +38,7 @@ func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel {
 		}
 	}
 
-	log.Debug("Token counts", "normal", counter[TOKEN_TYPE_NORMAL], "unknown", counter[TOKEN_TYPE_UNKNOWN], "control", counter[TOKEN_TYPE_CONTROL],
+	slog.Debug("Token counts", "normal", counter[TOKEN_TYPE_NORMAL], "unknown", counter[TOKEN_TYPE_UNKNOWN], "control", counter[TOKEN_TYPE_CONTROL],
 		"user defined", counter[TOKEN_TYPE_USER_DEFINED], "unused", counter[TOKEN_TYPE_UNUSED], "byte", counter[TOKEN_TYPE_BYTE],
 		"max token len", maxTokenLen)
 
@@ -94,7 +91,7 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 			fragments = append(fragments[:i], append(middle, fragments[i+1:]...)...)
 		}
 	}
-	log.Trace("fragments", "frags", fragments)
+	slog.Debug("fragments", "frags", fragments)
 
 	var ids []int32
 	for _, frag := range fragments {
@@ -132,7 +129,7 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 				}
 			}
 
-			log.Trace("tokenizer", "merges", merges)
+			slog.Debug("tokenizer", "merges", merges)
 
 			pairwise := func(a, b int) *candidate {
 				if a < 0 || b >= len(runes) {
@@ -159,7 +156,7 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 			pqv := pq.Values()
 			for _, v := range pqv {
 				e := v.(*candidate)
-				log.Trace("candidate", "candidate", e)
+				slog.Debug("candidate", "candidate", e)
 			}
 
 			for !pq.Empty() {
@@ -167,7 +164,7 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 				pair := v.(*candidate)
 				left, right := merges[pair.a], merges[pair.b]
 
-				log.Trace("pair", "left", left, "right", right)
+				slog.Debug("pair", "left", left, "right", right)
 				if len(left.runes) == 0 || len(right.runes) == 0 {
 					continue
 				}
@@ -192,14 +189,14 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 				}
 			}
 
-			log.Trace("merges", "merges", merges)
+			slog.Debug("merges", "merges", merges)
 
 			for _, merge := range merges {
 				if len(merge.runes) > 0 {
 					if id := spm.vocab.Encode(string(merge.runes)); id >= 0 {
 						ids = append(ids, id)
 					} else {
-						log.Error("missing token", "token", string(merge.runes))
+						slog.Debug("missing token", "token", string(merge.runes))
 					}
 				}
 			}
@@ -209,19 +206,19 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
 	if addSpecial && len(ids) > 0 {
 		if spm.vocab.AddBOS {
 			if ids[0] == spm.vocab.BOS {
-				log.Warn("adding bos token to prompt which already has it", "id", spm.vocab.BOS)
+				slog.Warn("adding bos token to prompt which already has it", "id", spm.vocab.BOS)
 			}
 
-			log.Debug("adding bos token to prompt", "id", spm.vocab.BOS)
+			slog.Debug("adding bos token to prompt", "id", spm.vocab.BOS)
 			ids = append([]int32{spm.vocab.BOS}, ids...)
 		}
 
 		if spm.vocab.AddEOS {
 			if ids[len(ids)-1] == spm.vocab.EOS {
-				log.Warn("adding eos token to prompt which already has it", "id", spm.vocab.EOS)
+				slog.Warn("adding eos token to prompt which already has it", "id", spm.vocab.EOS)
 			}
 
-			log.Debug("adding eos token to prompt", "id", spm.vocab.EOS)
+			slog.Debug("adding eos token to prompt", "id", spm.vocab.EOS)
 			ids = append(ids, spm.vocab.EOS)
 		}
 	}
@@ -244,6 +241,6 @@ func (spm SentencePieceModel) Decode(ids []int32) (string, error) {
 		}
 	}
 
-	log.Debug("decoded", "ids", ids, "text", sb.String())
+	slog.Debug("decoded", "ids", ids, "text", sb.String())
 	return sb.String(), nil
 }

readline/readline.go

@@ -116,9 +116,19 @@ func (i *Instance) Readline() (string, error) {
 
 			switch r {
 			case KeyUp:
-				i.historyPrev(buf, &currentLineBuf)
+				if i.History.Pos > 0 {
+					if i.History.Pos == i.History.Size() {
+						currentLineBuf = []rune(buf.String())
+					}
+					buf.Replace([]rune(i.History.Prev()))
+				}
 			case KeyDown:
-				i.historyNext(buf, &currentLineBuf)
+				if i.History.Pos < i.History.Size() {
+					buf.Replace([]rune(i.History.Next()))
+					if i.History.Pos == i.History.Size() {
+						buf.Replace(currentLineBuf)
+					}
+				}
 			case KeyLeft:
 				buf.MoveLeft()
 			case KeyRight:
@@ -175,10 +185,6 @@ func (i *Instance) Readline() (string, error) {
 			esc = true
 		case CharInterrupt:
 			return "", ErrInterrupt
-		case CharPrev:
-			i.historyPrev(buf, &currentLineBuf)
-		case CharNext:
-			i.historyNext(buf, &currentLineBuf)
 		case CharLineStart:
 			buf.MoveToStart()
 		case CharLineEnd:
@@ -240,24 +246,6 @@ func (i *Instance) HistoryDisable() {
 	i.History.Enabled = false
 }
 
-func (i *Instance) historyPrev(buf *Buffer, currentLineBuf *[]rune) {
-	if i.History.Pos > 0 {
-		if i.History.Pos == i.History.Size() {
-			*currentLineBuf = []rune(buf.String())
-		}
-		buf.Replace([]rune(i.History.Prev()))
-	}
-}
-
-func (i *Instance) historyNext(buf *Buffer, currentLineBuf *[]rune) {
-	if i.History.Pos < i.History.Size() {
-		buf.Replace([]rune(i.History.Next()))
-		if i.History.Pos == i.History.Size() {
-			buf.Replace(*currentLineBuf)
-		}
-	}
-}
-
 func NewTerminal() (*Terminal, error) {
 	fd := os.Stdin.Fd()
 	termios, err := SetRawMode(fd)

runner/ollamarunner/runner.go

@@ -691,6 +691,65 @@ type EmbeddingResponse struct {
 	Embedding []float32 `json:"embedding"`
 }
 
+func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) {
+	var req EmbeddingRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, fmt.Sprintf("bad request: %s", err), http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	slog.Debug("embedding request", "content", req.Content)
+
+	seq, err := s.NewSequence(req.Content, nil, NewSequenceParams{embedding: true})
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Failed to create new sequence: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	// Ensure there is a place to put the sequence, released when removed from s.seqs
+	if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
+		if errors.Is(err, context.Canceled) {
+			slog.Info("aborting embeddings request due to client closing the connection")
+		} else {
+			slog.Error("Failed to acquire semaphore", "error", err)
+		}
+		return
+	}
+
+	s.mu.Lock()
+	found := false
+	for i, sq := range s.seqs {
+		if sq == nil {
+			seq.cache, seq.inputs, err = s.cache.LoadCacheSlot(seq.inputs, req.CachePrompt)
+			if err != nil {
+				s.mu.Unlock()
+				http.Error(w, fmt.Sprintf("Failed to load cache: %v", err), http.StatusInternalServerError)
+				return
+			}
+			s.seqs[i] = seq
+			s.cond.Signal()
+			found = true
+			break
+		}
+	}
+	s.mu.Unlock()
+
+	if !found {
+		http.Error(w, "could not find an available sequence", http.StatusInternalServerError)
+		return
+	}
+
+	embedding := <-seq.embedding
+
+	if err := json.NewEncoder(w).Encode(&EmbeddingResponse{
+		Embedding: embedding,
+	}); err != nil {
+		http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
+	}
+}
+
 type HealthResponse struct {
 	Status   string  `json:"status"`
 	Progress float32 `json:"progress"`
@@ -868,13 +927,9 @@ func Execute(args []string) error {
 	defer listener.Close()
 
 	mux := http.NewServeMux()
-	// TODO: support embeddings
+	mux.HandleFunc("/embedding", server.embeddings)
-	mux.HandleFunc("POST /embedding", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/completion", server.completion)
-		http.Error(w, "this model does not support embeddings", http.StatusNotImplemented)
+	mux.HandleFunc("/health", server.health)
-	})
-
-	mux.HandleFunc("POST /completion", server.completion)
-	mux.HandleFunc("GET /health", server.health)
 
 	httpServer := http.Server{
 		Handler: mux,

sample/samplers.go

@@ -87,8 +87,8 @@ func (s *Sampler) sample(tokens []token) (token, error) {
 	// topK also sorts the tokens in descending order of logits
 	tokens = topK(tokens, s.topK)
 
+	// token logit values are updated to probabilities
 	tokens = temperature(tokens, s.temperature)
-	tokens = softmax(tokens)
 
 	tokens = topP(tokens, s.topP)
 	tokens = minP(tokens, s.minP)

sample/transforms.go

@@ -25,18 +25,8 @@ func (h *tokenHeap) Pop() any {
 	return x
 }
 
-// temperature applies scaling to the logits
+// temperature applies scaling and softmax to the logits
 func temperature(ts []token, temp float32) []token {
-	// Ensure temperature clipping near 0 to avoid numerical instability
-	temp = max(temp, 1e-7)
-	for i := range ts {
-		ts[i].value = ts[i].value / temp
-	}
-	return ts
-}
-
-// softmax applies normalization to the logits
-func softmax(ts []token) []token {
 	// Find max logit for numerical stability
 	maxLogit := float32(math.Inf(-1))
 	for _, t := range ts {
@@ -45,14 +35,15 @@ func softmax(ts []token) []token {
 		}
 	}
 
-	// Compute exp(x - max)
+	// Apply temperature and compute exp(x - max)
+	temp = max(temp, 1e-7)
 	var sum float32
 	for i, v := range ts {
-		ts[i].value = float32(math.Exp(float64(v.value - maxLogit)))
+		ts[i].value = float32(math.Exp(float64((v.value - maxLogit) / temp)))
 		sum += ts[i].value
 	}
 
-	// exp(x - max) / sum(exp(x - max))
+	// Normalize
 	for i := range ts {
 		ts[i].value /= sum
 	}

sample/transforms_test.go

@@ -32,83 +32,27 @@ func compareLogits(t *testing.T, name string, want []float32, got []token) {
 	}
 }
 
-func TestTemperature(t *testing.T) {
+func TestTemperatureAndSoftmax(t *testing.T) {
-	input := []float32{1.0, 4.0, -2.0, 0.0}
+	input := []float32{1, 4, -2, 0}
 	got := temperature(toTokens(input), 0.5)
-	want := []float32{2.0, 8.0, -4.0, 0.0}
-	compareLogits(t, "temperature(0.5)", want, got)
-
-	got = temperature(toTokens(input), 1.0)
-	want = []float32{1.0, 4.0, -2.0, 0.0}
-	compareLogits(t, "temperature(1)", want, got)
-
-	got = temperature(toTokens(input), 0.0)
-	want = []float32{1e7, 4e7, -2e7, 0.0}
-	compareLogits(t, "temperature(0)", want, got)
-}
-
-func TestSoftmax(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    []float32
-		expected []float32
-	}{
-		{
-			name:     "correctness softmax",
-			input:    []float32{1, -2, 3, 0},
-			expected: []float32{0.113550, 0.005653, 0.839024, 0.041773},
-		},
-		{
-			name:  "normal distribution",
-			input: []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367},
-		},
-		{
-			name:  "single value",
-			input: []float32{1.0},
-		},
-		{
-			name:  "identical values",
-			input: []float32{0.9, 0.9, 0.9},
-		},
-		{
-			name:  "large values",
-			input: []float32{1000.0, 2000.0, 3000.0},
-		},
-		{
-			name:  "small values",
-			input: []float32{1e-6, 2e-6, 3e-6},
-		},
-		{
-			name:  "negative values",
-			input: []float32{-1.0, -2.0, -3.0},
-		},
-		{
-			name:  "mixed values",
-			input: []float32{-100.0, 0.0, 100.0},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := softmax(toTokens(tt.input))
-
-			if tt.expected != nil {
-				compareLogits(t, tt.name, tt.expected, got)
-				return
-			}
 
 	// Check probabilities sum to 1
 	var sum float32
 	for _, token := range got {
 		sum += token.value
-				if token.value < 0 || token.value > 1 {
-					t.Errorf("probability out of range [0,1]: got %f", token.value)
-				}
 	}
 	if math.Abs(float64(sum-1.0)) > 1e-6 {
 		t.Errorf("probabilities don't sum to 1: got %f", sum)
 	}
-		})
+
+	got = temperature(toTokens(input), 1)
+	// Check probabilities sum to 1
+	sum = 0.0
+	for _, token := range got {
+		sum += token.value
+	}
+	if math.Abs(float64(sum-1.0)) > 1e-6 {
+		t.Errorf("probabilities don't sum to 1: got %f", sum)
 	}
 }
 
@@ -153,7 +97,7 @@ func TestTopP(t *testing.T) {
 	tokens := toTokens(input)
 
 	// First apply temperature and softmax to get probabilities
-	tokens = softmax(tokens)
+	tokens = temperature(tokens, 1)
 	tokens = topK(tokens, 20)
 
 	// Then apply topP
@@ -171,7 +115,7 @@ func TestMinP(t *testing.T) {
 	tokens := toTokens(input)
 
 	// First apply temperature and softmax
-	tokens = softmax(tokens)
+	tokens = temperature(tokens, 1)
 
 	// Then apply minP
 	got := minP(tokens, 0.2)
@@ -219,14 +163,6 @@ func BenchmarkTransforms(b *testing.B) {
 		}
 	})
 
-	b.Run("Softmax", func(b *testing.B) {
-		b.ResetTimer()
-		for b.Loop() {
-			copy(tokensCopy, tokens)
-			softmax(tokensCopy)
-		}
-	})
-
 	b.Run("TopK", func(b *testing.B) {
 		b.ResetTimer()
 		for b.Loop() {

server/routes.go

@@ -435,7 +435,7 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 		return
 	}
 
-	kvData, _, err := getModelData(m.ModelPath, false)
+	kvData, err := getKVData(m.ModelPath, false)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -483,7 +483,8 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 	}
 
 	if err := g.Wait(); err != nil {
-		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
+		slog.Error("embedding generation failed", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
 		return
 	}
 
@@ -544,7 +545,8 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
 
 	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
 	if err != nil {
-		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
+		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embedding: %v", err)})
 		return
 	}
 
@@ -848,23 +850,16 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
 	fmt.Fprint(&sb, m.String())
 	resp.Modelfile = sb.String()
 
-	kvData, tensors, err := getModelData(m.ModelPath, req.Verbose)
+	kvData, err := getKVData(m.ModelPath, req.Verbose)
 	if err != nil {
 		return nil, err
 	}
-
 	delete(kvData, "general.name")
 	delete(kvData, "tokenizer.chat_template")
 	resp.ModelInfo = kvData
 
-	tensorData := make([]api.Tensor, len(tensors.Items()))
-	for cnt, t := range tensors.Items() {
-		tensorData[cnt] = api.Tensor{Name: t.Name, Type: t.Type(), Shape: t.Shape}
-	}
-	resp.Tensors = tensorData
-
 	if len(m.ProjectorPaths) > 0 {
-		projectorData, _, err := getModelData(m.ProjectorPaths[0], req.Verbose)
+		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
 		if err != nil {
 			return nil, err
 		}
@@ -874,17 +869,17 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
 	return resp, nil
 }
 
-func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
+func getKVData(digest string, verbose bool) (ggml.KV, error) {
 	maxArraySize := 0
 	if verbose {
 		maxArraySize = -1
 	}
-	data, err := llm.LoadModel(digest, maxArraySize)
+	kvData, err := llm.LoadModel(digest, maxArraySize)
 	if err != nil {
-		return nil, ggml.Tensors{}, err
+		return nil, err
 	}
 
-	kv := data.KV()
+	kv := kvData.KV()
 
 	if !verbose {
 		for k := range kv {
@@ -894,7 +889,7 @@ func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
 		}
 	}
 
-	return kv, data.Tensors(), nil
+	return kv, nil
 }
 
 func (s *Server) ListHandler(c *gin.Context) {