llama: sync

CMakePresets.json

@@ -21,14 +21,16 @@
       "name": "CUDA 11",
       "inherits": [ "CUDA" ],
       "cacheVariables": {
-        "CMAKE_CUDA_ARCHITECTURES": "50;52;53;60;61;70;75;80;86"
+        "CMAKE_CUDA_ARCHITECTURES": "50;52;53;60;61;70;75;80;86",
+        "CMAKE_CUDA_FLAGS": "-Wno-deprecated-gpu-targets"
       }
     },
     {
       "name": "CUDA 12",
       "inherits": [ "CUDA" ],
       "cacheVariables": {
-        "CMAKE_CUDA_ARCHITECTURES": "50;60;61;70;75;80;86;87;89;90;90a;120"
+        "CMAKE_CUDA_ARCHITECTURES": "50;60;61;70;75;80;86;87;89;90;90a;120",
+        "CMAKE_CUDA_FLAGS": "-Wno-deprecated-gpu-targets"
       }
     },
     {

cmd/cmd.go

@@ -22,7 +22,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -32,7 +31,6 @@ import (
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/sync/errgroup"
 	"golang.org/x/term"
 
 	"github.com/ollama/ollama/api"
@@ -108,7 +106,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 	}
 	spinner.Stop()
 
-	req.Model = args[0]
+	req.Name = args[0]
 	quantize, _ := cmd.Flags().GetString("quantize")
 	if quantize != "" {
 		req.Quantize = quantize
@@ -119,43 +117,26 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	var mu sync.Mutex
+	if len(req.Files) > 0 {
-	var g errgroup.Group
+		fileMap := map[string]string{}
-	g.SetLimit(max(runtime.GOMAXPROCS(0)-1, 1))
+		for f, digest := range req.Files {
-	// copy files since we'll be modifying the map
-	temp := req.Files
-	req.Files = make(map[string]string, len(temp))
-	for f, digest := range temp {
-		g.Go(func() error {
 			if _, err := createBlob(cmd, client, f, digest, p); err != nil {
 				return err
 			}
-
+			fileMap[filepath.Base(f)] = digest
-			mu.Lock()
+		}
-			req.Files[filepath.Base(f)] = digest
+		req.Files = fileMap
-			mu.Unlock()
-			return nil
-		})
 	}
 
-	// copy files since we'll be modifying the map
+	if len(req.Adapters) > 0 {
-	temp = req.Adapters
+		fileMap := map[string]string{}
-	req.Adapters = make(map[string]string, len(temp))
+		for f, digest := range req.Adapters {
-	for f, digest := range temp {
-		g.Go(func() error {
 			if _, err := createBlob(cmd, client, f, digest, p); err != nil {
 				return err
 			}
-
+			fileMap[filepath.Base(f)] = digest
-			mu.Lock()
+		}
-			req.Adapters[filepath.Base(f)] = digest
+		req.Adapters = fileMap
-			mu.Unlock()
-			return nil
-		})
-	}
-
-	if err := g.Wait(); err != nil {
-		return err
 	}
 
 	bars := make(map[string]*progress.Bar)
@@ -232,7 +213,7 @@ func createBlob(cmd *cobra.Command, client *api.Client, path string, digest stri
 		}
 	}()
 
-	if err := client.CreateBlob(cmd.Context(), digest, io.TeeReader(bin, &pw)); err != nil {
+	if err = client.CreateBlob(cmd.Context(), digest, io.TeeReader(bin, &pw)); err != nil {
 		return "", err
 	}
 	return digest, nil

cmd/cmd_test.go

@@ -690,7 +690,7 @@ func TestCreateHandler(t *testing.T) {
 						return
 					}
 
-					if req.Model != "test-model" {
+					if req.Name != "test-model" {
 						t.Errorf("expected model name 'test-model', got %s", req.Name)
 					}
 

convert/convert.go

@@ -4,9 +4,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log/slog"
-	"os"
+	"slices"
 	"strings"
 
 	"github.com/ollama/ollama/fs/ggml"
@@ -84,14 +85,6 @@ func (ModelParameters) specialTokenTypes() []string {
 	}
 }
 
-func (ModelParameters) writeFile(f *os.File, kv ggml.KV, ts []ggml.Tensor) error {
-	return ggml.WriteGGUF(f, kv, ts)
-}
-
-func (AdapterParameters) writeFile(f *os.File, kv ggml.KV, ts []ggml.Tensor) error {
-	return ggml.WriteGGUF(f, kv, ts)
-}
-
 type ModelConverter interface {
 	// KV maps parameters to LLM key-values
 	KV(*Tokenizer) ggml.KV
@@ -103,8 +96,6 @@ type ModelConverter interface {
 
 	// specialTokenTypes returns any special token types the model uses
 	specialTokenTypes() []string
-	// writeFile writes the model to the provided io.WriteSeeker
-	writeFile(*os.File, ggml.KV, []ggml.Tensor) error
 }
 
 type moreParser interface {
@@ -119,11 +110,9 @@ type AdapterConverter interface {
 	// Replacements returns a list of string pairs to replace in tensor names.
 	// See [strings.Replacer](https://pkg.go.dev/strings#Replacer) for details
 	Replacements() []string
-
-	writeFile(*os.File, ggml.KV, []ggml.Tensor) error
 }
 
-func ConvertAdapter(fsys fs.FS, f *os.File, baseKV ggml.KV) error {
+func ConvertAdapter(fsys fs.FS, ws io.WriteSeeker, baseKV ggml.KV) error {
 	bts, err := fs.ReadFile(fsys, "adapter_config.json")
 	if err != nil {
 		return err
@@ -158,14 +147,14 @@ func ConvertAdapter(fsys fs.FS, f *os.File, baseKV ggml.KV) error {
 		return err
 	}
 
-	return conv.writeFile(f, conv.KV(baseKV), conv.Tensors(ts))
+	return writeFile(ws, conv.KV(baseKV), conv.Tensors(ts))
 }
 
 // Convert writes an Ollama compatible model to the provided io.WriteSeeker based on configurations
 // and files it finds in the input path.
 // Supported input model formats include safetensors.
 // Supported input tokenizers files include tokenizer.json (preferred) and tokenizer.model.
-func ConvertModel(fsys fs.FS, f *os.File) error {
+func ConvertModel(fsys fs.FS, ws io.WriteSeeker) error {
 	bts, err := fs.ReadFile(fsys, "config.json")
 	if err != nil {
 		return err
@@ -248,5 +237,13 @@ func ConvertModel(fsys fs.FS, f *os.File) error {
 		return err
 	}
 
-	return conv.writeFile(f, conv.KV(t), conv.Tensors(ts))
+	return writeFile(ws, conv.KV(t), conv.Tensors(ts))
+}
+
+func writeFile(ws io.WriteSeeker, kv ggml.KV, ts []ggml.Tensor) error {
+	for i := range ts {
+		ts[i].Shape = slices.Clone(ts[i].Shape)
+		slices.Reverse(ts[i].Shape)
+	}
+	return ggml.WriteGGUF(ws, kv, ts)
 }

fs/ggml/gguf.go

@@ -9,12 +9,8 @@ import (
 	"io"
 	"log/slog"
 	"maps"
-	"os"
-	"runtime"
 	"slices"
 	"strings"
-
-	"golang.org/x/sync/errgroup"
 )
 
 type containerGGUF struct {
@@ -506,22 +502,22 @@ func writeGGUFArray[S ~[]E, E any](w io.Writer, t uint32, s S) error {
 	return binary.Write(w, binary.LittleEndian, s)
 }
 
-func WriteGGUF(f *os.File, kv KV, ts []Tensor) error {
+func WriteGGUF(ws io.WriteSeeker, kv KV, ts []Tensor) error {
 	alignment := kv.Uint("general.alignment", 32)
 
-	if err := binary.Write(f, binary.LittleEndian, []byte("GGUF")); err != nil {
+	if err := binary.Write(ws, binary.LittleEndian, []byte("GGUF")); err != nil {
 		return err
 	}
 
-	if err := binary.Write(f, binary.LittleEndian, uint32(3)); err != nil {
+	if err := binary.Write(ws, binary.LittleEndian, uint32(3)); err != nil {
 		return err
 	}
 
-	if err := binary.Write(f, binary.LittleEndian, uint64(len(ts))); err != nil {
+	if err := binary.Write(ws, binary.LittleEndian, uint64(len(ts))); err != nil {
 		return err
 	}
 
-	if err := binary.Write(f, binary.LittleEndian, uint64(len(kv))); err != nil {
+	if err := binary.Write(ws, binary.LittleEndian, uint64(len(kv))); err != nil {
 		return err
 	}
 
@@ -529,7 +525,7 @@ func WriteGGUF(f *os.File, kv KV, ts []Tensor) error {
 	slices.Sort(keys)
 
 	for _, key := range keys {
-		if err := ggufWriteKV(f, key, kv[key]); err != nil {
+		if err := ggufWriteKV(ws, key, kv[key]); err != nil {
 			return err
 		}
 	}
@@ -545,34 +541,21 @@ func WriteGGUF(f *os.File, kv KV, ts []Tensor) error {
 	})
 
 	var s uint64
-	for i := range ts {
+	for _, t := range ts {
-		ts[i].Offset = s + uint64(ggufPadding(int64(s), int64(alignment)))
+		t.Offset = s + uint64(ggufPadding(int64(s), int64(alignment)))
-		if err := ggufWriteTensorInfo(f, ts[i]); err != nil {
+		if err := ggufWriteTensorInfo(ws, t); err != nil {
 			return err
 		}
-		s += ts[i].Size()
+		s += t.Size()
 	}
 
-	offset, err := f.Seek(0, io.SeekCurrent)
-	if err != nil {
-		return err
-	}
-
-	offset += ggufPadding(offset, int64(alignment))
-	slog.Debug("gguf", "offset", offset, "size", s, "alignment", alignment)
-
-	var g errgroup.Group
-	g.SetLimit(runtime.GOMAXPROCS(0))
 	for _, t := range ts {
-		t := t
+		if err := ggufWriteTensor(ws, t, int64(alignment)); err != nil {
-		w := io.NewOffsetWriter(f, offset+int64(t.Offset))
-		g.Go(func() error {
-			_, err := t.WriteTo(w)
 			return err
-		})
+		}
 	}
 
-	return g.Wait()
+	return nil
 }
 
 func ggufWriteKV(ws io.WriteSeeker, k string, v any) error {
@@ -644,8 +627,8 @@ func ggufWriteTensorInfo(ws io.WriteSeeker, t Tensor) error {
 		return err
 	}
 
-	for i := range len(t.Shape) {
+	for _, n := range t.Shape {
-		if err := binary.Write(ws, binary.LittleEndian, t.Shape[len(t.Shape)-i-1]); err != nil {
+		if err := binary.Write(ws, binary.LittleEndian, n); err != nil {
 			return err
 		}
 	}
@@ -657,6 +640,20 @@ func ggufWriteTensorInfo(ws io.WriteSeeker, t Tensor) error {
 	return binary.Write(ws, binary.LittleEndian, t.Offset)
 }
 
+func ggufWriteTensor(ws io.WriteSeeker, t Tensor, alignment int64) error {
+	offset, err := ws.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return err
+	}
+
+	if err := binary.Write(ws, binary.LittleEndian, bytes.Repeat([]byte{0}, int(ggufPadding(offset, alignment)))); err != nil {
+		return err
+	}
+
+	_, err = t.WriteTo(ws)
+	return err
+}
+
 func ggufPadding(offset, align int64) int64 {
 	return (align - offset%align) % align
 }

llama/llama.cpp/src/llama-grammar.cpp

@@ -907,7 +907,6 @@ llama_grammar_candidates llama_grammar_reject_candidates_for_stack(
 
 struct llama_grammar * llama_grammar_init_impl(
         const struct llama_vocab * vocab,
-        const struct ollama_vocab * ollama_vocab,
         const llama_grammar_element ** rules,
         size_t n_rules,
         size_t start_rule_index) {
@@ -963,7 +962,6 @@ struct llama_grammar * llama_grammar_init_impl(
     // then the pointers would be invalidated when the local vec_rules goes out of scope.
     return new llama_grammar {
         vocab,
-        ollama_vocab,
         std::move(vec_rules),
         std::move(stacks),
         /* .partial_utf8 = */     {},
@@ -977,7 +975,6 @@ struct llama_grammar * llama_grammar_init_impl(
 
 struct llama_grammar * llama_grammar_init_impl(
         const struct llama_vocab * vocab,
-        const struct ollama_vocab * ollama_vocab,
                       const char * grammar_str,
                       const char * grammar_root,
                               bool lazy,
@@ -1070,7 +1067,6 @@ struct llama_grammar * llama_grammar_init_impl(
     // then the pointers would be invalidated when the local vec_rules goes out of scope.
     return new llama_grammar {
         vocab,
-        ollama_vocab,
         std::move(vec_rules),
         std::move(stacks),
         /* .partial_utf8 = */     {},
@@ -1093,7 +1089,6 @@ void llama_grammar_free_impl(struct llama_grammar * grammar) {
 struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar) {
     auto * result = new llama_grammar {
         grammar.vocab,
-        grammar.o_vocab,
         grammar.rules,
         grammar.stacks,
         grammar.partial_utf8,
@@ -1121,6 +1116,7 @@ struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & gra
 }
 
 void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) {
+    GGML_ASSERT(grammar.vocab != nullptr);
 
     if (grammar.awaiting_trigger) {
         return;
@@ -1142,13 +1138,9 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
 
     for (size_t i = 0; i < cur_p->size; ++i) {
         const llama_token id      = cur_p->data[i].id;
-        const std::string piece = grammar.o_vocab ?
+        const std::string & piece = grammar.vocab->token_to_piece(id);
-            grammar.o_vocab->token_to_piece(id) :
-            grammar.vocab->token_to_piece(id);
 
-        const bool is_eog = grammar.o_vocab ? grammar.o_vocab->is_eog(id) : grammar.vocab->is_eog(id);
+        if (grammar.vocab->is_eog(id)) {
-
-        if (is_eog) {
             if (!allow_eog) {
                 cur_p->data[i].logit = -INFINITY;
             }
@@ -1167,10 +1159,9 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
 }
 
 void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token) {
+    GGML_ASSERT(grammar.vocab != nullptr);
 
-    const std::string piece = grammar.o_vocab ?
+    const auto & piece = grammar.vocab->token_to_piece(token);
-        grammar.o_vocab->token_to_piece(token) :
-        grammar.vocab->token_to_piece(token);
 
     if (grammar.awaiting_trigger) {
         if (std::find(grammar.trigger_tokens.begin(), grammar.trigger_tokens.end(), token) != grammar.trigger_tokens.end()) {
@@ -1200,14 +1191,13 @@ void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token
         }
     }
 
-    const bool is_eog = grammar.o_vocab ? grammar.o_vocab->is_eog(token) : grammar.vocab->is_eog(token);
+    if (grammar.vocab->is_eog(token)) {
-    if (is_eog) {
         for (const auto & stack : grammar.stacks) {
             if (stack.empty()) {
                 return;
             }
         }
-        GGML_ABORT("grammar error: end of grammar token received but grammar stack is not empty");
+        GGML_ABORT("fatal error");
     }
 
     llama_grammar_accept_str(grammar, piece);
@@ -1227,28 +1217,3 @@ void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string
         throw std::runtime_error("Unexpected empty grammar stack after accepting piece: " + piece);
     }
 }
-
-
-const std::string & ollama_vocab::token_to_piece(const uint32_t token) const {
-    try {
-        return token_to_piece_map.at(token);
-    } catch (const std::out_of_range&) {
-        throw std::runtime_error("Token not found in vocabulary: " + std::to_string(token));
-    }
-}
-
-void ollama_vocab::add_token_pieces(const uint32_t* tokens, size_t n_tokens, const char** pieces) {
-    for (size_t i = 0; i < n_tokens; i++) {
-        token_to_piece_map[tokens[i]] = pieces[i];
-    }
-}
-
-bool ollama_vocab::is_eog(const uint32_t token) const {
-    return special_eog_ids.count(token) > 0;
-}
-
-void ollama_vocab::set_eog_tokens(const uint32_t* tokens, size_t n_tokens) {
-    for (size_t i = 0; i < n_tokens; i++) {
-        special_eog_ids.insert(tokens[i]);
-    }
-}

llama/llama.cpp/src/llama-grammar.h

@@ -6,19 +6,8 @@
 #include <regex>
 #include <string>
 #include <vector>
-#include <set>
 
 struct llama_vocab;
-struct ollama_vocab {
-    std::map<uint32_t, std::string> token_to_piece_map;
-    std::set<uint32_t> special_eog_ids;
-
-    const std::string & token_to_piece(const uint32_t token) const;
-    void add_token_pieces(const uint32_t* tokens, size_t n_tokens, const char** pieces);
-    void set_eog_tokens(const uint32_t* tokens, size_t n_tokens);
-    bool is_eog(const uint32_t token) const;
-
-};
 
 // grammar element type
 enum llama_gretype {
@@ -125,7 +114,6 @@ struct llama_grammar_trigger_pattern {
 struct llama_grammar {
     // note: allow null vocab for testing (not great)
     const llama_vocab * vocab;
-    const ollama_vocab * o_vocab;
 
     const llama_grammar_rules  rules;  // TODO: shared ptr
           llama_grammar_stacks stacks;
@@ -153,14 +141,12 @@ struct llama_grammar {
 // note: needed for tests (not great)
 struct llama_grammar * llama_grammar_init_impl(
         const struct llama_vocab * vocab,
-        const struct ollama_vocab * ollama_vocab,
         const llama_grammar_element ** rules,
         size_t n_rules,
         size_t start_rule_index);
 
 struct llama_grammar * llama_grammar_init_impl(
         const struct llama_vocab * vocab,
-        const struct ollama_vocab * ollama_vocab,
                       const char * grammar_str,
                       const char * grammar_root,
                               bool lazy,

llama/llama.cpp/src/llama-sampling.cpp

@@ -1465,7 +1465,7 @@ static void llama_sampler_grammar_reset(struct llama_sampler * smpl) {
         trigger_patterns_c.push_back(trigger_pattern.pattern.c_str());
     }
 
-    auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, nullptr, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
+    auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
                                                  ctx->grammar->lazy, trigger_patterns_c.data(), trigger_patterns_c.size(),
                                                  ctx->grammar->trigger_tokens.data(), ctx->grammar->trigger_tokens.size());
 
@@ -1547,7 +1547,7 @@ static struct llama_sampler * llama_sampler_init_grammar_impl(
             /* .vocab        = */ vocab,
             /* .grammar_str  = */ grammar_str,
             /* .grammar_root = */ grammar_root,
-            /* .grammar      = */ llama_grammar_init_impl(vocab, nullptr, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens),
+            /* .grammar      = */ llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens),
         };
         if (!ctx->grammar) {
             delete ctx;

progress/bar.go

@@ -64,7 +64,7 @@ func formatDuration(d time.Duration) string {
 func (b *Bar) String() string {
 	termWidth, _, err := term.GetSize(int(os.Stderr.Fd()))
 	if err != nil {
-		termWidth = defaultTermWidth
+		termWidth = 80
 	}
 
 	var pre strings.Builder

progress/progress.go

@@ -4,16 +4,8 @@ import (
 	"bufio"
 	"fmt"
 	"io"
-	"os"
 	"sync"
 	"time"
-
-	"golang.org/x/term"
-)
-
-const (
-	defaultTermWidth  = 80
-	defaultTermHeight = 24
 )
 
 type State interface {
@@ -91,11 +83,6 @@ func (p *Progress) Add(key string, state State) {
 }
 
 func (p *Progress) render() {
-	_, termHeight, err := term.GetSize(int(os.Stderr.Fd()))
-	if err != nil {
-		termHeight = defaultTermHeight
-	}
-
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
@@ -115,9 +102,8 @@ func (p *Progress) render() {
 	fmt.Fprint(p.w, "\033[1G")
 
 	// render progress lines
-	maxHeight := min(len(p.states), termHeight)
+	for i, state := range p.states {
-	for i := len(p.states) - maxHeight; i < len(p.states); i++ {
+		fmt.Fprint(p.w, state.String(), "\033[K")
-		fmt.Fprint(p.w, p.states[i].String(), "\033[K")
 		if i < len(p.states)-1 {
 			fmt.Fprint(p.w, "\n")
 		}